refactor vault role (#2733)

* Move front-proxy-client certs back to kube mount

We want the same CA for all k8s certs

* Refactor vault to use a third party module

The module adds idempotency and reduces some of the repetitive
logic in the vault role

Requires ansible-modules-hashivault on ansible node and hvac
on the vault hosts themselves

Add upgrade test scenario
Remove bootstrap-os tags from tasks

* fix upgrade issues

* improve unseal logic

* specify ca and fix etcd check

* Fix initialization check

bump machine size

roles/vault/tasks/shared/auth_backend.yml

@@ -1,20 +1,10 @@
 ---
-- name: shared/auth_backend | Test if the auth backend exists
-  uri:
-    url: "{{ vault_leader_url }}/v1/sys/auth/{{ auth_backend_path }}/tune"
-    headers: "{{ vault_headers }}"
-    validate_certs: false
-  ignore_errors: true
-  register: vault_auth_backend_check
-
-- name: shared/auth_backend | Add the cert auth backend if needed
-  uri:
-    url: "{{ vault_leader_url }}/v1/sys/auth/{{ auth_backend_path }}"
-    headers: "{{ vault_headers }}"
-    method: POST
-    body_format: json
-    body:
-      description: "{{ auth_backend_description|d('') }}"
-      type: "{{ auth_backend_type }}"
-    status_code: 204
-  when: vault_auth_backend_check|failed
+- name: shared/auth_backend | Enable auth backend {{ auth_backend_path }}
+  hashivault_auth_enable:
+    url: "{{ vault_leader_url }}"
+    token: "{{ vault_root_token }}"
+    ca_cert: "{{ vault_cert_dir }}/ca.pem"
+    name: "{{ auth_backend_type }}"
+    mount_point: "{{ auth_backend_path }}"
+    description: "{{ auth_backend_description|d('') }}"
+  register: result

roles/vault/tasks/shared/cert_auth_mount.yml

@@ -10,11 +10,10 @@
         max_lease_ttl: "{{ vault_max_lease_ttl }}"
 
 - name: shared/auth_mount | Create a dummy role for issuing certs from auth-pki
-  uri:
-    url: "{{ hostvars[groups.vault|first]['vault_leader_url'] }}/v1/auth-pki/roles/dummy"
-    headers: "{{ hostvars[groups.vault|first]['vault_headers'] }}"
-    method: POST
-    body_format: json
-    body:
-      {'allow_any_name': true}
-    status_code: 204
+  hashivault_approle_role_create:
+    url: "{{ vault_leader_url }}"
+    token: "{{ vault_root_token }}"
+    ca_cert: "{{ vault_cert_dir }}/ca.pem"
+    name: "auth-pki/roles/dummy"
+    policies:
+      allow_any_name: true

roles/vault/tasks/shared/check_etcd.yml

@@ -4,9 +4,12 @@
   uri:
     url: "{{ vault_etcd_url }}/health"
     validate_certs: no
+    client_cert: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}.pem"
+    client_key: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}-key.pem"
+
     return_content: yes
   until: vault_etcd_health_check.status == 200 or vault_etcd_health_check.status == 401
-  retries: 10
+  retries: 3
   delay: 2
   delegate_to: "{{groups['etcd'][0]}}"
   run_once: true

roles/vault/tasks/shared/check_vault.yml

@@ -8,24 +8,42 @@
 
 # Check if vault is reachable on the localhost
 - name: check_vault | Attempt to pull local https Vault health
-  uri:
-    url: "{{ vault_config.listener.tcp.tls_disable|d()|ternary('http', 'https') }}://localhost:{{ vault_port }}/v1/sys/health"
-    headers: "{{ vault_client_headers }}"
-    status_code: 200,429,500,501,503
-    validate_certs: no
-  ignore_errors: true
-  register: vault_local_service_health
+  command: /bin/true
+  notify: wait for vault up nowait
+
+- meta: flush_handlers
 
 - name: check_vault | Set facts about local Vault health
   set_fact:
-    vault_is_running: "{{ vault_local_service_health|succeeded }}"
-    vault_is_initialized: "{{ vault_local_service_health.get('json', {}).get('initialized', false) }}"
-    vault_is_sealed: "{{ vault_local_service_health.get('json', {}).get('sealed', true) }}"
-    # vault_in_standby: "{{ vault_local_service_health.get('json', {}).get('standby', true) }}"
+    vault_is_running: "{{ vault_health_check.get('status', '-1') in vault_successful_http_codes }}"
+
+- name: check_vault | Set facts about local Vault health
+  set_fact:
+    vault_is_initialized: "{{ vault_health_check.get('json', {}).get('initialized', false) }}"
+    vault_is_sealed: "{{ vault_health_check.get('json', {}).get('sealed', true) }}"
+    # vault_in_standby: "{{ vault_health_check.get('json', {}).get('standby', true) }}"
     # vault_run_version: "{{ vault_local_service_health.get('json', {}).get('version', '') }}"
 
+- name: check_vault | Check is vault is initialized in etcd if vault is not running
+  command: |-
+    curl \
+      --cacert {{ etcd_cert_dir }}/ca.pem \
+      --cert {{ etcd_cert_dir}}/node-{{ inventory_hostname }}.pem \
+      --key {{ etcd_cert_dir }}/node-{{ inventory_hostname }}-key.pem \
+      -X POST -d '{"key": "{{ "/vault/core/seal-config" | b64encode }}"}' \
+      {{ etcd_access_addresses.split(',') | first }}/v3alpha/kv/range
+  register: vault_etcd_exists
+  retries: 4
+  delay: "{{ retry_stagger | random + 3 }}"
+  run_once: true
+  when: not vault_is_running and vault_etcd_available
+  changed_when: false
+
 - name: check_vault | Set fact about the Vault cluster's initialization state
   set_fact:
-    vault_cluster_is_initialized: "{{ vault_is_initialized or hostvars[item]['vault_is_initialized'] }}"
+    vault_cluster_is_initialized: >-
+      {{ vault_is_initialized or
+         hostvars[item]['vault_is_initialized'] or
+         'Key not found' not in vault_etcd_exists.stdout|default('Key not found') }}
   with_items: "{{ groups.vault }}"
   run_once: true

roles/vault/tasks/shared/config_ca.yml

@@ -4,26 +4,26 @@
   register: vault_ca_cert_cat
 
 - name: config_ca | Pull current CA cert from Vault
-  uri:
-    url: "{{ vault_leader_url }}/v1/{{ config_ca_mount_path }}/ca/pem"
-    headers: "{{ vault_headers }}"
-    return_content: true
-    status_code: 200,204
-    validate_certs: no
+  hashivault_read:
+    url: "{{ vault_leader_url }}"
+    token: "{{ vault_root_token }}"
+    ca_cert: "{{ vault_cert_dir }}/ca.pem"
+    secret: "{{ config_ca_mount_path }}/ca"
+    key: "pem"
   register: vault_pull_current_ca
+  failed_when: false
 
 - name: config_ca | Read root CA key for Vault
   command: "cat {{ config_ca_ca_key }}"
   register: vault_ca_key_cat
-  when: vault_ca_cert_cat.stdout.strip() != vault_pull_current_ca.content.strip()
+  when: vault_ca_cert_cat.stdout.strip() != vault_pull_current_ca.get("data","").strip()
 
 - name: config_ca | Configure pki mount to use the found root CA cert and key
-  uri:
-    url: "{{ vault_leader_url }}/v1/{{ config_ca_mount_path }}/config/ca"
-    headers: "{{ vault_headers }}"
-    method: POST
-    body_format: json
-    body:
+  hashivault_write:
+    url: "{{ vault_leader_url }}"
+    token: "{{ vault_root_token }}"
+    ca_cert: "{{ vault_cert_dir }}/ca.pem"
+    secret: "{{ config_ca_mount_path }}/config/ca"
+    data:
       pem_bundle: "{{ vault_ca_cert_cat.stdout + '\n' + vault_ca_key_cat.stdout }}"
-    status_code: 204
-  when: vault_ca_cert_cat.stdout.strip() != vault_pull_current_ca.get("content","").strip()
+  when: vault_ca_cert_cat.stdout.strip() != vault_pull_current_ca.get("data","").strip()

roles/vault/tasks/shared/create_role.yml

@@ -1,42 +1,36 @@
 ---
-# The JSON inside JSON here is intentional (Vault API wants it)
-- name: create_role | Create a policy for the new role allowing issuing
-  uri:
-    url: "{{ hostvars[groups.vault|first]['vault_leader_url'] }}/v1/sys/policy/{{ create_role_name }}"
-    headers: "{{ hostvars[groups.vault|first]['vault_headers'] }}"
-    method: PUT
-    body_format: json
-    body:
-      rules: >-
-             {%- if create_role_policy_rules|d("default") == "default" -%}
-             {{
-             { 'path': {
-                 create_role_mount_path + '/issue/' + create_role_name: {'policy': 'write'},
-                 create_role_mount_path + '/roles/' + create_role_name: {'policy': 'read'}
-             }} | to_json + '\n'
-             }}
-             {%- else -%}
-             {{ create_role_policy_rules | to_json + '\n' }}
-             {%- endif -%}
-    status_code: 204
-  delegate_to: "{{ groups.vault|first }}"
-  run_once: true
+- name: create_role | Create a policy for the new role
+  hashivault_policy_set:
+    url: "{{ vault_leader_url }}"
+    token: "{{ vault_root_token }}"
+    ca_cert: "{{ vault_cert_dir }}/ca.pem"
+    name: "{{ create_role_name }}"
+    rules: >-
+           {%- if create_role_policy_rules|d("default") == "default" -%}
+           {{
+           { 'path': {
+                create_role_mount_path + '/issue/' + create_role_name: {'policy': 'write'},
+                create_role_mount_path + '/roles/' + create_role_name: {'policy': 'read'}
+            }} | to_json + '\n'
+            }}
+            {%- else -%}
+            {{ create_role_policy_rules | to_json + '\n' }}
+            {%- endif -%}
 
 - name: create_role | Create {{ create_role_name }} role in the {{ create_role_mount_path }} pki mount
-  uri:
-    url: "{{ hostvars[groups.vault|first]['vault_leader_url'] }}/v1/{{ create_role_mount_path }}/roles/{{ create_role_name }}"
-    headers: "{{ hostvars[groups.vault|first]['vault_headers'] }}"
-    method: POST
-    body_format: json
-    body: >-
-          {%- if create_role_options|d("default") == "default" -%}
-          {'allow_any_name': true}
-          {%- else -%}
-          {{ create_role_options }}
-          {%- endif -%}
-    status_code: 204
-  delegate_to: "{{ groups.vault|first }}"
-  run_once: true
+  hashivault_write:
+    url: "{{ vault_leader_url }}"
+    token: "{{ vault_root_token }}"
+    ca_cert: "{{ vault_cert_dir }}/ca.pem"
+    secret: "{{ create_role_mount_path }}/roles/{{ create_role_name }}"
+    data: |
+     {%- if create_role_options|d("default") == "default" -%}
+     {
+     allow_any_name: true
+     }
+     {%- else -%}
+     {{ create_role_options | to_json }}
+     {%- endif -%}
 
 ## Userpass based auth method
 

roles/vault/tasks/shared/find_leader.yml

@@ -5,16 +5,16 @@
     url: "{{ vault_config.listener.tcp.tls_disable|d()|ternary('http', 'https') }}://localhost:{{ vault_port }}/v1/sys/health"
     headers: "{{ hostvars[groups.vault|first]['vault_headers'] }}"
     method: HEAD
-    status_code: 200,429,503
+    status_code: 200,429,501,503
   register: vault_leader_check
   until: "vault_leader_check|succeeded"
   retries: 10
 
 - name: find_leader | Set fact for current http leader
   set_fact:
-    vault_leader_url: "{{ vault_config.listener.tcp.tls_disable|d()|ternary('http', 'https') }}://{{ item }}:{{ vault_port }}"
+    vault_leader_url: "{{ vault_config.listener.tcp.tls_disable|d()|ternary('http', 'https') }}://{{ inventory_hostname }}:{{ vault_port }}"
   with_items: "{{ groups.vault }}"
-  when: "hostvars[item]['vault_leader_check'].get('status') in [200,503]"
+  when: "hostvars[item]['vault_leader_check'].get('status') in [200,501,503]"
   # run_once: true
 
 - name: find_leader| show vault_leader_url

roles/vault/tasks/shared/gen_ca.yml

@@ -1,35 +1,38 @@
 ---
-- name: "bootstrap/gen_ca | Ensure cert_dir {{ gen_ca_cert_dir }} exists"
+- name: "bootstrap/gen_ca | Ensure cert_dir {{ gen_ca_cert_dir }} exists on necessary hosts"
   file:
     mode: 0755
     path: "{{ gen_ca_cert_dir }}"
     state: directory
+  delegate_to: "{{ item }}"
+  with_items: "{{ (groups[gen_ca_copy_group|default('vault')]) | union(groups['vault']) }}"
 
 - name: "bootstrap/gen_ca | Generate {{ gen_ca_mount_path }} root CA"
-  uri:
-    url: "{{ vault_leader_url }}/v1/{{ gen_ca_mount_path }}/root/generate/exported"
-    headers: "{{ gen_ca_vault_headers }}"
-    method: POST
-    body_format: json
-    body: "{{ gen_ca_vault_options }}"
-    status_code: 200,204
-  register: vault_ca_gen
-  delegate_to: "{{ groups.vault|first }}"
+  hashivault_write:
+    url: "{{ vault_leader_url }}"
+    token: "{{ vault_root_token }}"
+    ca_cert: "{{ vault_cert_dir }}/ca.pem"
+    secret: "{{ gen_ca_mount_path }}/root/generate/exported"
+    data: "{{ gen_ca_vault_options }}"
   run_once: true
+  no_log: true
+  register: vault_ca_gen
 
 - name: "bootstrap/gen_ca | Copy {{ gen_ca_mount_path }} root CA cert locally"
   copy:
-    content: "{{ hostvars[groups.vault|first]['vault_ca_gen']['json']['data']['certificate'] }}"
+    content: "{{ vault_ca_gen['data']['data']['certificate'] }}"
     dest: "{{ gen_ca_cert_dir }}/ca.pem"
     mode: 0644
-  when: vault_ca_gen.status == 200
+  when: '"data" in vault_ca_gen.keys()'
+  delegate_to: "{{ item }}"
+  with_items: "{{ (groups[gen_ca_copy_group|default('vault')]) | union(groups['vault']) }}"
 
 
 - name: "bootstrap/gen_ca | Copy {{ gen_ca_mount_path }} root CA key to necessary hosts"
   copy:
-    content: "{{ hostvars[groups.vault|first]['vault_ca_gen']['json']['data']['private_key'] }}"
+    content: "{{ vault_ca_gen['data']['data']['private_key']}}"
     dest: "{{ gen_ca_cert_dir }}/ca-key.pem"
     mode: 0640
-  when: vault_ca_gen.status == 200
+  when: '"data" in vault_ca_gen.keys()'
   delegate_to: "{{ item }}"
   with_items: "{{ (groups[gen_ca_copy_group|default('vault')]) | union(groups['vault']) }}"

roles/vault/tasks/shared/gen_userpass.yml

@@ -1,16 +1,13 @@
 ---
 - name: shared/gen_userpass | Create the Username/Password combo for the role
-  uri:
-    url: "{{ hostvars[groups.vault|first]['vault_leader_url'] }}/v1/auth/userpass/users/{{ gen_userpass_username }}"
-    headers: "{{ hostvars[groups.vault|first]['vault_headers'] }}"
-    method: POST
-    body_format: json
-    body:
-      username: "{{ gen_userpass_username }}"
-      password: "{{ gen_userpass_password }}"
-      policies: "{{ gen_userpass_role }}"
-    status_code: 204
-  delegate_to: "{{ groups.vault|first }}"
+  hashivault_userpass_create:
+    url: "{{ vault_leader_url }}"
+    token: "{{ vault_root_token }}"
+    ca_cert: "{{ vault_cert_dir }}/ca.pem"
+    name: "{{ gen_userpass_username }}"
+    pass: "{{ gen_userpass_password }}"
+    policies:
+      - "{{ gen_userpass_role }}"
   run_once: true
 
 - name: shared/gen_userpass | Ensure destination directory exists

roles/vault/tasks/shared/issue_cert.yml

@@ -39,52 +39,58 @@
   delegate_to: "{{ groups.vault|first }}"
   run_once: true
 
-- name: gen_certs_vault | Log into Vault and obtain an token
-  uri:
-    url: "{{ hostvars[groups.vault|first]['vault_leader_url'] }}/v1/auth/userpass/login/{{ user_vault_creds.username }}"
-    headers:
-      Accept: application/json
-      Content-Type: application/json
-    method: POST
-    body_format: json
-    body:
-      password: "{{ user_vault_creds.password }}"
-  register: vault_login_result
-  delegate_to: "{{ groups.vault|first }}"
+- name: gen_certs_vault | Ensure vault cert dir exists
+  file:
+    path: "{{ vault_cert_dir }}"
+    state: directory
+    recurse: yes
+    owner: "vault"
+    group: "vault"
+    mode: 0755
+
+- name: gen_certs_vault | install hvac
+  pip:
+    name: "hvac"
+    state: "present"
+
+- name: gen_certs_vault | Pull vault CA
+  get_url:
+    url: "{{ issue_cert_url }}/v1/vault/ca/pem"
+    dest: "{{ vault_cert_dir }}/ca.pem"
+    validate_certs: no
+  when: '"https" in issue_cert_url'
+
+- name: gen_certs_vault | Log into Vault and obtain a scoped token
+  hashivault_token_create:
+    url: "{{ issue_cert_url }}"
+    token: "{{ vault_root_token | default(hostvars[groups.vault|first]['vault_root_token']) }}"
+    ca_cert: "{{ vault_cert_dir }}/ca.pem"
+    policies: "{{ user_vault_creds.username }}"
+    display_name: "{{ user_vault_creds.username }}"
+  register: vault_client_token_request
   run_once: true
 
-- name: gen_certs_vault | Set fact for vault_client_token
+- name: gen_certs_vault | Pull token from request
   set_fact:
-    vault_client_token: "{{ vault_login_result.get('json', {}).get('auth', {}).get('client_token') }}"
+    vault_client_token: "{{ vault_client_token_request['token']['auth']['client_token'] }}"
   run_once: true
 
-- name: gen_certs_vault | Set fact for Vault API token
-  set_fact:
-    issue_cert_headers:
-      Accept: application/json
-      Content-Type: application/json
-      X-Vault-Token: "{{ vault_client_token }}"
-  run_once: true
-  when: vault_client_token != ""
-
 - name: "issue_cert | Generate {{ issue_cert_path }} for {{ issue_cert_role }} role"
-  uri:
-    url: "{{ issue_cert_url }}/v1/{{ issue_cert_mount_path|d('pki') }}/issue/{{ issue_cert_role }}"
-    headers: "{{ issue_cert_headers }}"
-    method: POST
-    body_format: json
-    body:
+  hashivault_write:
+    url: "{{ issue_cert_url }}"
+    token: "{{ vault_client_token }}"
+    ca_cert: "{% if 'https' in issue_cert_url %}{{ vault_cert_dir }}/ca.pem{% endif %}"
+    secret: "{{ issue_cert_mount_path|d('/pki') }}/issue/{{ issue_cert_role }}"
+    data:
       alt_names: "{{ issue_cert_alt_names | d([]) | join(',') }}"
       common_name: "{{ issue_cert_common_name | d(issue_cert_path.rsplit('/', 1)[1].rsplit('.', 1)[0]) }}"
       format: "{{ issue_cert_format | d('pem') }}"
       ip_sans: "{{ issue_cert_ip_sans | default([]) | join(',') }}"
   register: issue_cert_result
-  delegate_to: "{{ issue_cert_hosts|first }}"
-  run_once: true
 
 - name: "issue_cert | Copy {{ issue_cert_path }} cert to all hosts"
   copy:
-    content: "{{ issue_cert_result['json']['data']['certificate'] }}\n"
+    content: "{{ issue_cert_result['data']['data']['certificate'] }}\n"
     dest: "{{ issue_cert_path }}"
     group: "{{ issue_cert_file_group | d('root' )}}"
     mode: "{{ issue_cert_file_mode | d('0644') }}"
@@ -92,7 +98,7 @@
 
 - name: "issue_cert | Copy key for {{ issue_cert_path }} to all hosts"
   copy:
-    content: "{{ issue_cert_result['json']['data']['private_key'] }}"
+    content: "{{ issue_cert_result['data']['data']['private_key'] }}"
     dest: "{{ issue_cert_path.rsplit('.', 1)|first }}-key.{{ issue_cert_path.rsplit('.', 1)|last }}"
     group: "{{ issue_cert_file_group | d('root' )}}"
     mode: "{{ issue_cert_file_mode | d('0640') }}"
@@ -100,7 +106,7 @@
 
 - name: issue_cert | Copy issuing CA cert
   copy:
-    content: "{{ issue_cert_result['json']['data']['issuing_ca'] }}\n"
+    content: "{{ issue_cert_result['data']['data']['issuing_ca'] }}\n"
     dest: "{{ issue_cert_path | dirname }}/{{ issue_cert_ca_filename | default('ca.pem') }}"
     group: "{{ issue_cert_file_group | d('root' )}}"
     mode: "{{ issue_cert_file_mode | d('0644') }}"
@@ -109,7 +115,7 @@
 
 - name: issue_cert | Copy certificate serial to all hosts
   copy:
-    content: "{{ issue_cert_result['json']['data']['serial_number'] }}"
+    content: "{{ issue_cert_result['data']['data']['serial_number'] }}"
     dest: "{{ issue_cert_path.rsplit('.', 1)|first }}.serial"
     group: "{{ issue_cert_file_group | d('root' )}}"
     mode: "{{ issue_cert_file_mode | d('0640') }}"

roles/vault/tasks/shared/pki_mount.yml

@@ -1,27 +1,12 @@
 ---
-- name: "shared/mount | Test if {{ pki_mount_path }} PKI mount exists"
-  uri:
-    url: "{{ vault_leader_url }}/v1/sys/mounts/{{ pki_mount_path }}/tune"
-    headers: "{{ vault_headers }}"
-  ignore_errors: true
-  register: vault_pki_mount_check
 
-- name: shared/mount | Set pki mount type
-  set_fact:
-    mount_options: "{{ pki_mount_options | combine({'type': 'pki'}) }}"
-  when: vault_pki_mount_check|failed
-
-- name: shared/mount | Mount {{ pki_mount_path }} PKI mount if needed
-  uri:
-    url: "{{ vault_leader_url }}/v1/sys/mounts/{{ pki_mount_path }}"
-    headers: "{{ vault_headers }}"
-    method: POST
-    body_format: json
-    body: "{{ mount_options|d() }}"
-    status_code: 204
-  when: vault_pki_mount_check|failed
-
-- name: shared/mount | Unset mount options
-  set_fact:
-    mount_options: {}
-  when: vault_pki_mount_check|failed
+- name: shared/mount | Enable {{ pki_mount_path }} PKI mount
+  hashivault_secret_enable:
+    url: "{{ vault_leader_url }}"
+    token: "{{ vault_root_token }}"
+    ca_cert: "{{ vault_cert_dir }}/ca.pem"
+    name: "{{ pki_mount_path }}"
+    backend: "pki"
+    config: "{{ pki_mount_options }}"
+  register: secret_enable_result
+  failed_when: 'secret_enable_result.rc !=0 and "existing mount" not in secret_enable_result.msg'

roles/vault/tasks/shared/sync.yml

@@ -1,5 +1,4 @@
 ---
-
 - name: "sync_file | Cat the file"
   command: "cat {{ sync_file_path }}"
   register: sync_file_cat

roles/vault/tasks/shared/sync_file.yml

@@ -1,7 +1,6 @@
 ---
 
 # NOTE: This should be a role (or custom module), but currently include_role is too buggy to use
-
 - name: "sync_file | Set facts for directory and file when sync_file_path is defined"
   set_fact:
     sync_file_dir: "{{ sync_file_path | dirname }}"