Enhancing vault integration

Added persistent storage

Auto-create vault and awx via playbooks

Create a new pattern for custom containers where we can do initialization

Auto-install roles needed for plumbing via the Makefile

tools/docker-compose/ansible/host_vars/localhost.yml

@@ -0,0 +1,2 @@
+---
+sources_dest: '../_sources'

tools/docker-compose/ansible/initialize_containers.yml

@@ -0,0 +1,10 @@
+---
+- name: Run any pre-hooks for other container
+  hosts: localhost
+  gather_facts: false
+  tasks:
+    - name: Initialize vault
+      include_role:
+        name: vault
+        tasks_from: initialize
+      when: enable_vault | bool

tools/docker-compose/ansible/plumb_vault.yml

@@ -0,0 +1,8 @@
+---
+- name: Plumb AWX for Vault
+  hosts: localhost
+  gather_facts: False
+  tasks:
+    - include_role:
+        name: vault
+        tasks_from: plumb

tools/docker-compose/ansible/requirements.yml

@@ -0,0 +1,5 @@
+---
+collections:
+  - awx.awx
+  - flowerysong.hvault
+  - community.docker

tools/docker-compose/ansible/roles/sources/defaults/main.yml

@@ -1,5 +1,4 @@
 ---
-sources_dest: '../_sources'
 compose_name: 'docker-compose.yml'
 awx_image: 'ghcr.io/ansible/awx_devel'
 pg_port: 5432

tools/docker-compose/ansible/roles/sources/tasks/main.yml

@@ -101,10 +101,6 @@
   include_tasks: ldap.yml
   when: enable_ldap | bool
 
-- name: Include Vault tasks if enabled
-  include_tasks: vault.yaml
-  when: enable_vault | bool
-
 - name: Render Docker-Compose
   template:
     src: docker-compose.yml.j2

tools/docker-compose/ansible/roles/sources/tasks/vault.yaml

@@ -1,20 +0,0 @@
----
-- name: create vault secret file and scope into ansible-runtime
-  block:
-    - ansible.builtin.stat:
-        path: "{{ sources_dest }}/secrets/{{ item }}.yml"
-      register: vault_secret
-      loop:
-        - vault_password
-
-    - ansible.builtin.template:
-        src: "secrets.yml.j2"
-        dest: "{{ sources_dest }}/secrets/{{ item.item }}.yml"
-        mode: "0600"
-      loop: "{{ vault_secret.results }}"
-      loop_control:
-        label: "{{ item.item }}"
-
-    - include_vars: "{{ sources_dest }}/secrets/{{ item.item }}.yml"
-      loop: "{{ vault_secret.results }}"
-      no_log: true

tools/docker-compose/ansible/roles/sources/templates/docker-compose.yml.j2

@@ -235,16 +235,18 @@ services:
 {% endif %}
 {% if enable_vault|bool %}
   vault:
-    image: hashicorp/vault:latest
+    image: hashicorp/vault:1.14
     container_name: tools_vault_1
+    command: server
     hostname: vault
     ports:
       - "1234:1234"
     environment:
-      VAULT_DEV_ROOT_TOKEN_ID: "{{ vault_password }}"
-      VAULT_DEV_LISTEN_ADDRESS: "0.0.0.0:1234"
+      VAULT_LOCAL_CONFIG: '{"storage": {"file": {"path": "/vault/file"}}, "listener": [{"tcp": { "address": "0.0.0.0:1234", "tls_disable": true}}], "default_lease_ttl": "168h", "max_lease_ttl": "720h", "ui": true}'
     cap_add:
       - IPC_LOCK
+    volumes:
+      - 'hashicorp_vault_data:/vault/file'
 {% endif %}
 
 volumes:
@@ -260,6 +262,10 @@ volumes:
     name: tools_ldap_1
     driver: local
 {% endif %}
+{% if enable_vault|bool %}
+  hashicorp_vault_data:
+    name: tools_vault_1
+{% endif %}
 {% if enable_prometheus|bool %}
   prometheus_storage:
     name: tools_prometheus_storage

tools/docker-compose/ansible/roles/vault/defaults/main.yml

@@ -0,0 +1,2 @@
+---
+vault_file: "{{ sources_dest }}/secrets/vault_init.yml"

tools/docker-compose/ansible/roles/vault/tasks/initialize.yml

@@ -0,0 +1,62 @@
+---
+- name: See if vault has been initialized
+  ansible.builtin.stat:
+    path: "{{ vault_file }}"
+  register: vault_secret_file_info
+
+- block:
+    - name: Start the vault
+      community.docker.docker_compose:
+        state: present
+        services: vault
+        project_src: "{{ sources_dest }}"
+
+    - name: Run the initialization
+      community.docker.docker_container_exec:
+        command: vault operator init
+        container: tools_vault_1
+        env:
+          VAULT_ADDR: "http://127.0.0.1:1234"
+      register: vault_initialization
+
+    - name: Write out initialization file
+      copy:
+        # lines 1-4 are the keys, 6 is the root token
+        content: |
+          {{ vault_initialization.stdout_lines[0] | regex_replace('Unseal Key ', 'Unseal_Key_') }}
+          {{ vault_initialization.stdout_lines[1] | regex_replace('Unseal Key ', 'Unseal_Key_') }}
+          {{ vault_initialization.stdout_lines[2] | regex_replace('Unseal Key ', 'Unseal_Key_') }}
+          {{ vault_initialization.stdout_lines[3] | regex_replace('Unseal Key ', 'Unseal_Key_') }}
+          {{ vault_initialization.stdout_lines[4] | regex_replace('Unseal Key ', 'Unseal_Key_') }}
+          {{ vault_initialization.stdout_lines[6] | regex_replace('Initial Root Token', 'Initial_Root_Token') }}
+        dest: "{{ vault_file }}"
+
+    - name: Unlock the vault
+      include_role:
+        name: vault
+        tasks_from: unseal.yml
+
+    - name: Create an engine
+      flowerysong.hvault.engine:
+        path: "my_engine"
+        type: "kv"
+        vault_addr: "http://localhost:1234"
+        token: "{{ Initial_Root_Token }}"
+      register: engine
+
+    - name: Create a secret
+      flowerysong.hvault.kv:
+        mount_point: "my_engine/my_root"
+        key: "my_folder"
+        value:
+          my_key: "this_is_the_secret_value"
+        vault_addr: "http://localhost:1234"
+        token: "{{ Initial_Root_Token }}"
+
+  always:
+    - name: Stop the vault
+      community.docker.docker_compose:
+        state: absent
+        project_src: "{{ sources_dest }}"
+
+  when: not vault_secret_file_info.stat.exists

tools/docker-compose/ansible/roles/vault/tasks/plumb.yml

@@ -0,0 +1,56 @@
+---
+- name: Load vault keys
+  include_vars:
+    file: "{{ vault_file }}"
+
+- name: Create a HashiCorp Vault Credential
+  awx.awx.credential:
+    credential_type: HashiCorp Vault Secret Lookup
+    name: Vault Lookup Cred
+    organization: Default
+    inputs:
+      api_version: "v1"
+      cacert: ""
+      default_auth_path: "approle"
+      kubernetes_role: ""
+      namespace: ""
+      role_id: ""
+      secret_id: ""
+      token: "{{ Initial_Root_Token }}"
+      url: "http://tools_vault_1:1234"
+  register: vault_cred
+
+- name: Create a custom credential type
+  awx.awx.credential_type:
+    name: Vault Custom Cred Type
+    kind: cloud
+    injectors:
+      extra_vars:
+        the_secret_from_vault: "{{ '{{' }} password {{ '}}' }}"
+    inputs:
+      fields:
+        - type: "string"
+          id: "password"
+          label: "Password"
+          secret: true
+  register: custom_vault_cred_type
+
+- name: Create a credential of the custom type
+  awx.awx.credential:
+    credential_type: "{{ custom_vault_cred_type.id }}"
+    name: Credential From Vault
+    inputs: {}
+    organization: Default
+  register: custom_credential
+
+- name: Use the Vault Credential For the new credential
+  awx.awx.credential_input_source:
+    input_field_name: password
+    target_credential: "{{ custom_credential.id }}"
+    source_credential: "{{ vault_cred.id }}"
+    metadata:
+      auth_path: ""
+      secret_backend: "my_engine"
+      secret_key: "my_key"
+      secret_path: "/my_root/my_folder"
+      secret_version: ""

tools/docker-compose/ansible/roles/vault/tasks/unseal.yml

@@ -0,0 +1,14 @@
+---
+- name: Load vault keys
+  include_vars:
+    file: "{{ vault_file }}"
+
+- name: Unseal the vault
+  flowerysong.hvault.seal:
+    vault_addr: "http://localhost:1234"
+    state: unsealed
+    key: "{{ item }}"
+  loop:
+    - "{{ Unseal_Key_1 }}"
+    - "{{ Unseal_Key_2 }}"
+    - "{{ Unseal_Key_3 }}"

tools/docker-compose/ansible/unseal_vault.yml

@@ -0,0 +1,13 @@
+---
+- name: Run tasks post startup
+  hosts: localhost
+  gather_facts: False
+  tasks:
+    - name: Unseal the vault
+      include_role:
+        name: vault
+        tasks_from: unseal
+
+    - name: Display root token
+      debug:
+        var: Initial_Root_Token