Add TLS certificate auth for HashiCorp Vault (#14534)

* Add TLS certificate auth for HashiCorp Vault

Add support for AWX to authenticate with HashiCorp Vault using
TLS client certificates.

Also updates the documentation for the HashiCorp Vault secret management
plugins to include both the new TLS options and the missing Kubernetes
auth method options.

Signed-off-by: Andrew Austin <aaustin@redhat.com>

* Refactor docker-compose vault for TLS cert auth

Add TLS configuration to the docker-compose Vault configuration and
use that method by default in vault plumbing.

This ensures that the result of bringing up the docker-compose stack
with vault enabled and running the plumb-vault playbook is a fully
working credential retrieval setup using TLS client cert authentication.

Signed-off-by: Andrew Austin <aaustin@redhat.com>

* Remove incorrect trailing space

Co-authored-by: Hao Liu <44379968+TheRealHaoLiu@users.noreply.github.com>

* Make vault init idempotent

- improve error handling for vault_initialization
- ignore error if vault cert auth is already configured
- removed unused register

* Add VAULT_TLS option

Make TLS for HashiCorp Vault optional and configurable via VAULT_TLS env var

* Add retries for vault init

Sometime it took longer for vault to fully come up and init will fail

---------

Signed-off-by: Andrew Austin <aaustin@redhat.com>
Co-authored-by: Hao Liu <44379968+TheRealHaoLiu@users.noreply.github.com>
Co-authored-by: Hao Liu <haoli@redhat.com>

tools/docker-compose/ansible/plumb_vault.yml

@@ -2,6 +2,8 @@
 - name: Plumb AWX for Vault
   hosts: localhost
   gather_facts: False
+  vars:
+    awx_host: "https://127.0.0.1:8043"
   tasks:
     - include_role:
         name: vault

tools/docker-compose/ansible/roles/sources/defaults/main.yml

@@ -30,6 +30,24 @@ ldap_public_key_file: '{{ ldap_cert_dir }}/{{ ldap_public_key_file_name }}'
 ldap_private_key_file: '{{ ldap_cert_dir }}/{{ ldap_private_key_file_name }}'
 ldap_cert_subject: "/C=US/ST=NC/L=Durham/O=awx/CN="
 
+# Hashicorp Vault
+enable_vault: false
+vault_tls: false
+hashivault_cert_dir: '{{ sources_dest }}/vault_certs'
+hashivault_server_cert_subject: "/C=US/ST=NC/L=Durham/O=awx/CN=tools-vault-1"
+hashivault_server_cert_extensions:
+  - "subjectAltName = DNS:tools_vault_1, DNS:localhost"
+  - "keyUsage = digitalSignature, nonRepudiation"
+  - "extendedKeyUsage = serverAuth"
+hashivault_client_cert_extensions:
+  - "subjectAltName = DNS:awx-vault-client"
+  - "keyUsage = digitalSignature, nonRepudiation"
+  - "extendedKeyUsage = serverAuth, clientAuth"
+hashivault_client_cert_subject: "/C=US/ST=NC/L=Durham/O=awx/CN=awx-vault-client"
+hashivault_server_public_keyfile: '{{ hashivault_cert_dir }}/server.crt'
+hashivault_server_private_keyfile: '{{ hashivault_cert_dir }}/server.key'
+hashivault_client_public_keyfile: '{{ hashivault_cert_dir }}/client.crt'
+hashivault_client_private_keyfile: '{{ hashivault_cert_dir }}/client.key'
 # Metrics
 enable_splunk: false
 enable_grafana: false

tools/docker-compose/ansible/roles/sources/tasks/main.yml

@@ -101,6 +101,10 @@
   include_tasks: ldap.yml
   when: enable_ldap | bool
 
+- name: Include vault TLS tasks if enabled
+  include_tasks: vault_tls.yml
+  when: enable_vault | bool
+
 - name: Render Docker-Compose
   template:
     src: docker-compose.yml.j2

tools/docker-compose/ansible/roles/sources/tasks/vault_tls.yml

@@ -0,0 +1,31 @@
+---
+- name: Create Certificates for HashiCorp Vault
+  block:
+    - name: Create Hashicorp Vault cert directory
+      file:
+        path: "{{ hashivault_cert_dir }}"
+        state: directory
+
+    - name: Generate vault server certificate
+      command: 'openssl req -new -newkey rsa:2048 -x509 -days 365 -nodes -out {{ hashivault_server_public_keyfile }} -keyout {{ hashivault_server_private_keyfile }} -subj "{{ hashivault_server_cert_subject }}"{% for ext in hashivault_server_cert_extensions %} -addext "{{ ext }}"{% endfor %}'
+      args:
+        creates: "{{ hashivault_server_public_keyfile }}"
+
+    - name: Generate vault test client certificate
+      command: 'openssl req -new -newkey rsa:2048 -x509 -days 365 -nodes -out {{ hashivault_client_public_keyfile }} -keyout {{ hashivault_client_private_keyfile }} -subj "{{ hashivault_client_cert_subject }}"{% for ext in hashivault_client_cert_extensions %} -addext "{{ ext }}"{% endfor %}'
+      args:
+        creates: "{{ hashivault_client_public_keyfile }}"
+
+    - name: Set mode for vault certificates
+      ansible.builtin.file:
+        path: "{{ hashivault_cert_dir }}"
+        recurse: true
+        state: directory
+        mode: 0777
+  when: vault_tls | bool
+
+- name: Delete Certificates for HashiCorp Vault
+  file:
+    path: "{{ hashivault_cert_dir }}"
+    state: absent
+  when: vault_tls | bool == false

tools/docker-compose/ansible/roles/sources/templates/docker-compose.yml.j2

@@ -252,7 +252,7 @@ services:
     privileged: true
   {% endfor %}
 {% endif %}
-{% if enable_vault|bool %}
+{% if enable_vault | bool %}
   vault:
     image: hashicorp/vault:1.14
     container_name: tools_vault_1
@@ -261,10 +261,17 @@ services:
     ports:
       - "1234:1234"
     environment:
+{% if vault_tls | bool %}
+      VAULT_LOCAL_CONFIG: '{"storage": {"file": {"path": "/vault/file"}}, "listener": [{"tcp": { "address": "0.0.0.0:1234", "tls_disable": false, "tls_cert_file": "/vault/tls/server.crt", "tls_key_file": "/vault/tls/server.key"}}], "default_lease_ttl": "168h", "max_lease_ttl": "720h", "ui": true}'
+{% else %}
       VAULT_LOCAL_CONFIG: '{"storage": {"file": {"path": "/vault/file"}}, "listener": [{"tcp": { "address": "0.0.0.0:1234", "tls_disable": true}}], "default_lease_ttl": "168h", "max_lease_ttl": "720h", "ui": true}'
+{% endif %}
     cap_add:
       - IPC_LOCK
     volumes:
+{% if vault_tls | bool %}
+      - '../../docker-compose/_sources/vault_certs:/vault/tls'
+{% endif %}
       - 'hashicorp_vault_data:/vault/file'
 {% endif %}
 

tools/docker-compose/ansible/roles/vault/defaults/main.yml

@@ -1,2 +1,7 @@
 ---
 vault_file: "{{ sources_dest }}/secrets/vault_init.yml"
+admin_password_file: "{{ sources_dest }}/secrets/admin_password.yml"
+vault_cert_dir: '{{ sources_dest }}/vault_certs'
+vault_server_cert: "{{ vault_cert_dir }}/server.crt"
+vault_client_cert: "{{ vault_cert_dir }}/client.crt"
+vault_client_key: "{{ vault_cert_dir }}/client.key"

tools/docker-compose/ansible/roles/vault/tasks/initialize.yml

@@ -1,4 +1,7 @@
 ---
+- name: Set vault_addr
+  include_tasks: set_vault_addr.yml
+
 - block:
     - name: Start the vault
       community.docker.docker_compose:
@@ -12,9 +15,16 @@
         command: vault operator init
         container: tools_vault_1
         env:
-          VAULT_ADDR: "http://127.0.0.1:1234"
+          VAULT_ADDR: "{{ vault_addr }}"
+          VAULT_SKIP_VERIFY: "true"
       register: vault_initialization
-      ignore_errors: true
+      failed_when:
+        - vault_initialization.rc != 0
+        - vault_initialization.stderr.find("Vault is already initialized") == -1
+      changed_when:
+        - vault_initialization.rc == 0
+      retries: 5
+      delay: 5
 
     - name: Write out initialization file
       copy:
@@ -34,21 +44,52 @@
         name: vault
         tasks_from: unseal.yml
 
+    - name: Configure the vault with cert auth
+      block:
+        - name: Create a cert auth mount
+          flowerysong.hvault.write:
+            path: "sys/auth/cert"
+            vault_addr: "{{ vault_addr_from_host }}"
+            validate_certs: false
+            token: "{{ Initial_Root_Token }}"
+            data:
+              type: "cert"
+          register: vault_auth_cert
+          failed_when:
+            - vault_auth_cert.result.errors | default([]) | length > 0
+            - "'path is already in use at cert/' not in vault_auth_cert.result.errors | default([])"
+          changed_when:
+            - vault_auth_cert.result.errors | default([]) | length == 0
+
+        - name: Configure client certificate
+          flowerysong.hvault.write:
+            path: "auth/cert/certs/awx-client"
+            vault_addr: "{{ vault_addr_from_host }}"
+            validate_certs: false
+            token: "{{ Initial_Root_Token }}"
+            data:
+              name: awx-client
+              certificate: "{{ lookup('ansible.builtin.file', '{{ vault_client_cert }}') }}"
+              policies:
+                - root
+      when: vault_tls | bool
+
     - name: Create an engine
       flowerysong.hvault.engine:
         path: "my_engine"
         type: "kv"
-        vault_addr: "http://localhost:1234"
+        vault_addr: "{{ vault_addr_from_host }}"
+        validate_certs: false
         token: "{{ Initial_Root_Token }}"
-      register: engine
 
-    - name: Create a secret
+    - name: Create a demo secret
       flowerysong.hvault.kv:
         mount_point: "my_engine/my_root"
         key: "my_folder"
         value:
           my_key: "this_is_the_secret_value"
-        vault_addr: "http://localhost:1234"
+        vault_addr: "{{ vault_addr_from_host }}"
+        validate_certs: false
         token: "{{ Initial_Root_Token }}"
 
   always:

tools/docker-compose/ansible/roles/vault/tasks/plumb.yml

@@ -1,29 +1,45 @@
 ---
+- name: Set vault_addr
+  include_tasks: set_vault_addr.yml
+
 - name: Load vault keys
   include_vars:
     file: "{{ vault_file }}"
 
+- name: Get AWX admin password
+  include_vars:
+    file: "{{ admin_password_file }}"
+
 - name: Create a HashiCorp Vault Credential
   awx.awx.credential:
     credential_type: HashiCorp Vault Secret Lookup
     name: Vault Lookup Cred
     organization: Default
+    controller_host: "{{ awx_host }}"
+    controller_username: admin
+    controller_password: "{{ admin_password }}"
+    validate_certs: false
     inputs:
       api_version: "v1"
-      cacert: ""
-      default_auth_path: "approle"
+      cacert: "{{ lookup('ansible.builtin.file', '{{ vault_server_cert }}', errors='ignore') }}"
+      default_auth_path: "cert"
       kubernetes_role: ""
       namespace: ""
-      role_id: ""
-      secret_id: ""
+      client_cert_public: "{{ lookup('ansible.builtin.file', '{{ vault_client_cert }}', errors='ignore') }}"
+      client_cert_private: "{{ lookup('ansible.builtin.file', '{{ vault_client_key }}', errors='ignore') }}"
       token: "{{ Initial_Root_Token }}"
-      url: "http://tools_vault_1:1234"
+      url: "{{ vault_addr_from_container }}"
   register: vault_cred
 
 - name: Create a custom credential type
   awx.awx.credential_type:
     name: Vault Custom Cred Type
     kind: cloud
+    controller_host: "{{ awx_host }}"
+    controller_username: admin
+    controller_password: "{{ admin_password }}"
+
+    validate_certs: false
     injectors:
       extra_vars:
         the_secret_from_vault: "{{ '{{' }} password {{ '}}' }}"
@@ -38,6 +54,11 @@
 - name: Create a credential of the custom type
   awx.awx.credential:
     credential_type: "{{ custom_vault_cred_type.id }}"
+    controller_host: "{{ awx_host }}"
+    controller_username: admin
+    controller_password: "{{ admin_password }}"
+
+    validate_certs: false
     name: Credential From Vault
     inputs: {}
     organization: Default
@@ -48,6 +69,11 @@
     input_field_name: password
     target_credential: "{{ custom_credential.id }}"
     source_credential: "{{ vault_cred.id }}"
+    controller_host: "{{ awx_host }}"
+    controller_username: admin
+    controller_password: "{{ admin_password }}"
+
+    validate_certs: false
     metadata:
       auth_path: ""
       secret_backend: "my_engine"

tools/docker-compose/ansible/roles/vault/tasks/set_vault_addr.yml

@@ -0,0 +1,19 @@
+---
+- name: Detect if vault cert directory exist
+  stat:
+    path: "{{ vault_cert_dir }}"
+  register: vault_cert_dir_stat
+
+- name: Set vault_addr for http
+  set_fact:
+    vault_addr: "http://127.0.0.1:1234"
+    vault_addr_from_host: "http://localhost:1234"
+    vault_addr_from_container: "http://tools_vault_1:1234"
+  when: vault_cert_dir_stat.stat.exists == false
+
+- name: Set vault_addr for https
+  set_fact:
+    vault_addr: "https://127.0.0.1:1234"
+    vault_addr_from_host: "https://localhost:1234"
+    vault_addr_from_container: "https://tools_vault_1:1234"
+  when: vault_cert_dir_stat.stat.exists == true

tools/docker-compose/ansible/roles/vault/tasks/unseal.yml

@@ -1,11 +1,15 @@
 ---
+- name: Set vault_addr
+  include_tasks: set_vault_addr.yml
+
 - name: Load vault keys
   include_vars:
     file: "{{ vault_file }}"
 
 - name: Unseal the vault
   flowerysong.hvault.seal:
-    vault_addr: "http://localhost:1234"
+    vault_addr: "{{ vault_addr_from_host }}"
+    validate_certs: false
     state: unsealed
     key: "{{ item }}"
   loop: