Enhancing vault integration

Added persistent storage

Auto-create vault and awx via playbooks

Create a new pattern for custom containers where we can do initialization

Auto-install roles needed for plumbing via the Makefile

Makefile

@@ -532,6 +532,9 @@ docker-compose-sources: .git/hooks/pre-commit
             $(EXTRA_SOURCES_ANSIBLE_OPTS)
 
 docker-compose: awx/projects docker-compose-sources
+	ansible-galaxy install --ignore-certs -r tools/docker-compose/ansible/requirements.yml;
+	ansible-playbook -i tools/docker-compose/inventory tools/docker-compose/ansible/initialize_containers.yml \
+	  -e enable_vault=$(VAULT);
 	$(DOCKER_COMPOSE) -f tools/docker-compose/_sources/docker-compose.yml $(COMPOSE_OPTS) up $(COMPOSE_UP_OPTS) --remove-orphans
 
 docker-compose-credential-plugins: awx/projects docker-compose-sources
@@ -583,7 +586,7 @@ docker-clean:
 	-$(foreach image_id,$(shell docker images --filter=reference='*/*/*awx_devel*' --filter=reference='*/*awx_devel*' --filter=reference='*awx_devel*' -aq),docker rmi --force $(image_id);)
 
 docker-clean-volumes: docker-compose-clean docker-compose-container-group-clean
-	docker volume rm -f tools_awx_db tools_grafana_storage tools_prometheus_storage $(docker volume ls --filter name=tools_redis_socket_ -q)
+	docker volume rm -f tools_awx_db tools_vault_1 tools_grafana_storage tools_prometheus_storage $(docker volume ls --filter name=tools_redis_socket_ -q)
 
 docker-refresh: docker-clean docker-compose
 

tools/docker-compose/README.md

@@ -505,32 +505,45 @@ Run a HashiVault container alongside of AWX.
 VAULT=true make docker-compose
 ```
 
-Go to `http://localhost:1234` sign in with method "Token".
+You can find the initialization data at `tools/docker-compose/_sources/secrets/vault_init.yml`,
+This includes the unseal keys and a root token.
 
-You can find the generated token at `tools/docker-compose/_sources/secrets/vault_password.yml`,
+You will need to unseal the HashiVault each time the container is started.
-this is a root token, and it should not need a corresponding username.
+The easiest way to do that is to run:
-Note that the token will be different on each restart, as it is re-generated by the playbook,
+```bash
-and the container does not use a persistent volume.
+ansible-playbook tools/docker-compose/ansible/unseal_vault.yml
+```
+This will perform the unseal and also display the root token for login.
 
-As a demo, click "Enable new engine +", click "KV" and Next.
+For demo purposes, Vault will be auto-configured to include a Key Value (KV) vault called `my_engine` along with a secret called `my_key` in `/my_engine/my_root/my_folder`.
-In the "Path" enter "my_engine" and click "Enable Engine".
+The secret value is `this_is_the_secret_value`.
-Click on the name of the engine and then "Create secret +".
-In the "Path for this secret" enter "my_root/my_folder" and in the "Secret Data" put "my_key" for key and **"my_value"** for value.
 
-Then go to AWX and create a new HashiVault credential with the generated token.
+To create a secret connected to this vault in AWX you can run the following playbook:
-Then go to any other arbitrary credential and click the key icon on an input to use a credential lookup plugin.
+```bash
-In the "External Secret Management System" menu, first select the already-created HashiVault credential.
+export CONTROLLER_USERNAME=<your username>
+export CONTROLLER_PASSWORD=<your password>
+ansible-playbook tools/docker-compose/ansible/plumb_vault.yml
+```
 
-Then in the "Metadata" menu, put in this data which is important for the integration:
+This will create the following items in your AWX instance:
- - Name of Secret Backend: "my_engine"
+* A credential called `Vault Lookup Cred` tied to the vault instance.
- - Path to Secret: "data/my_root/my_folder"
+* A custom credential type called `Vault Custom Cred Type`.
- - Key Name: "my_key"
+* A credential called `Credential From Vault` which is of the created type using the `Vault Lookup Cred` to get the password.
 
-After this, apply the credential to a job template that writes the data in a debug task.
+The custom credential type adds a variable when used in a playbook called `the_secret_from_vault`.
-In the job output, you should see **my_value**.
+If you have a playbook like:
+```
+---
+- name: Show a vault secret
+  hosts: localhost
+  connection: local
+  gather_facts: False
+  tasks:
+    - debug:
+        var: the_secret_from_vault
+```
 
-(NOTE: the "arbitrary credential" could be a new custom credential type that injects to extra vars
+And run it through AWX with the credential `Credential From Vault` tied to it, the debug should result in `this_is_the_secret_value`
-which is used in corresponding playbook that prints hostvars, but this doc assumes you know how to do that)
 
 The extremely non-obvious input is the fact that the fact prefixes "data/" unexpectedly.
 This was discovered by inspecting the secret with the vault CLI, which may help with future troubleshooting.

tools/docker-compose/ansible/host_vars/localhost.yml

@@ -0,0 +1,2 @@
+---
+sources_dest: '../_sources'

tools/docker-compose/ansible/initialize_containers.yml

@@ -0,0 +1,10 @@
+---
+- name: Run any pre-hooks for other container
+  hosts: localhost
+  gather_facts: false
+  tasks:
+    - name: Initialize vault
+      include_role:
+        name: vault
+        tasks_from: initialize
+      when: enable_vault | bool

tools/docker-compose/ansible/plumb_vault.yml

@@ -0,0 +1,8 @@
+---
+- name: Plumb AWX for Vault
+  hosts: localhost
+  gather_facts: False
+  tasks:
+    - include_role:
+        name: vault
+        tasks_from: plumb

tools/docker-compose/ansible/requirements.yml

@@ -0,0 +1,5 @@
+---
+collections:
+  - awx.awx
+  - flowerysong.hvault
+  - community.docker

tools/docker-compose/ansible/roles/sources/defaults/main.yml

@@ -1,5 +1,4 @@
 ---
-sources_dest: '../_sources'
 compose_name: 'docker-compose.yml'
 awx_image: 'ghcr.io/ansible/awx_devel'
 pg_port: 5432

tools/docker-compose/ansible/roles/sources/tasks/main.yml

@@ -101,10 +101,6 @@
   include_tasks: ldap.yml
   when: enable_ldap | bool
 
-- name: Include Vault tasks if enabled
-  include_tasks: vault.yaml
-  when: enable_vault | bool
-
 - name: Render Docker-Compose
   template:
     src: docker-compose.yml.j2

tools/docker-compose/ansible/roles/sources/tasks/vault.yaml

@@ -1,20 +0,0 @@
----
-- name: create vault secret file and scope into ansible-runtime
-  block:
-    - ansible.builtin.stat:
-        path: "{{ sources_dest }}/secrets/{{ item }}.yml"
-      register: vault_secret
-      loop:
-        - vault_password
-
-    - ansible.builtin.template:
-        src: "secrets.yml.j2"
-        dest: "{{ sources_dest }}/secrets/{{ item.item }}.yml"
-        mode: "0600"
-      loop: "{{ vault_secret.results }}"
-      loop_control:
-        label: "{{ item.item }}"
-
-    - include_vars: "{{ sources_dest }}/secrets/{{ item.item }}.yml"
-      loop: "{{ vault_secret.results }}"
-      no_log: true

tools/docker-compose/ansible/roles/sources/templates/docker-compose.yml.j2

@@ -235,16 +235,18 @@ services:
 {% endif %}
 {% if enable_vault|bool %}
   vault:
-    image: hashicorp/vault:latest
+    image: hashicorp/vault:1.14
     container_name: tools_vault_1
+    command: server
     hostname: vault
     ports:
       - "1234:1234"
     environment:
-      VAULT_DEV_ROOT_TOKEN_ID: "{{ vault_password }}"
+      VAULT_LOCAL_CONFIG: '{"storage": {"file": {"path": "/vault/file"}}, "listener": [{"tcp": { "address": "0.0.0.0:1234", "tls_disable": true}}], "default_lease_ttl": "168h", "max_lease_ttl": "720h", "ui": true}'
-      VAULT_DEV_LISTEN_ADDRESS: "0.0.0.0:1234"
     cap_add:
       - IPC_LOCK
+    volumes:
+      - 'hashicorp_vault_data:/vault/file'
 {% endif %}
 
 volumes:
@@ -260,6 +262,10 @@ volumes:
     name: tools_ldap_1
     driver: local
 {% endif %}
+{% if enable_vault|bool %}
+  hashicorp_vault_data:
+    name: tools_vault_1
+{% endif %}
 {% if enable_prometheus|bool %}
   prometheus_storage:
     name: tools_prometheus_storage

tools/docker-compose/ansible/roles/vault/defaults/main.yml

@@ -0,0 +1,2 @@
+---
+vault_file: "{{ sources_dest }}/secrets/vault_init.yml"

tools/docker-compose/ansible/roles/vault/tasks/initialize.yml

@@ -0,0 +1,62 @@
+---
+- name: See if vault has been initialized
+  ansible.builtin.stat:
+    path: "{{ vault_file }}"
+  register: vault_secret_file_info
+
+- block:
+    - name: Start the vault
+      community.docker.docker_compose:
+        state: present
+        services: vault
+        project_src: "{{ sources_dest }}"
+
+    - name: Run the initialization
+      community.docker.docker_container_exec:
+        command: vault operator init
+        container: tools_vault_1
+        env:
+          VAULT_ADDR: "http://127.0.0.1:1234"
+      register: vault_initialization
+
+    - name: Write out initialization file
+      copy:
+        # lines 1-4 are the keys, 6 is the root token
+        content: |
+          {{ vault_initialization.stdout_lines[0] | regex_replace('Unseal Key ', 'Unseal_Key_') }}
+          {{ vault_initialization.stdout_lines[1] | regex_replace('Unseal Key ', 'Unseal_Key_') }}
+          {{ vault_initialization.stdout_lines[2] | regex_replace('Unseal Key ', 'Unseal_Key_') }}
+          {{ vault_initialization.stdout_lines[3] | regex_replace('Unseal Key ', 'Unseal_Key_') }}
+          {{ vault_initialization.stdout_lines[4] | regex_replace('Unseal Key ', 'Unseal_Key_') }}
+          {{ vault_initialization.stdout_lines[6] | regex_replace('Initial Root Token', 'Initial_Root_Token') }}
+        dest: "{{ vault_file }}"
+
+    - name: Unlock the vault
+      include_role:
+        name: vault
+        tasks_from: unseal.yml
+
+    - name: Create an engine
+      flowerysong.hvault.engine:
+        path: "my_engine"
+        type: "kv"
+        vault_addr: "http://localhost:1234"
+        token: "{{ Initial_Root_Token }}"
+      register: engine
+
+    - name: Create a secret
+      flowerysong.hvault.kv:
+        mount_point: "my_engine/my_root"
+        key: "my_folder"
+        value:
+          my_key: "this_is_the_secret_value"
+        vault_addr: "http://localhost:1234"
+        token: "{{ Initial_Root_Token }}"
+
+  always:
+    - name: Stop the vault
+      community.docker.docker_compose:
+        state: absent
+        project_src: "{{ sources_dest }}"
+
+  when: not vault_secret_file_info.stat.exists

tools/docker-compose/ansible/roles/vault/tasks/plumb.yml

@@ -0,0 +1,56 @@
+---
+- name: Load vault keys
+  include_vars:
+    file: "{{ vault_file }}"
+
+- name: Create a HashiCorp Vault Credential
+  awx.awx.credential:
+    credential_type: HashiCorp Vault Secret Lookup
+    name: Vault Lookup Cred
+    organization: Default
+    inputs:
+      api_version: "v1"
+      cacert: ""
+      default_auth_path: "approle"
+      kubernetes_role: ""
+      namespace: ""
+      role_id: ""
+      secret_id: ""
+      token: "{{ Initial_Root_Token }}"
+      url: "http://tools_vault_1:1234"
+  register: vault_cred
+
+- name: Create a custom credential type
+  awx.awx.credential_type:
+    name: Vault Custom Cred Type
+    kind: cloud
+    injectors:
+      extra_vars:
+        the_secret_from_vault: "{{ '{{' }} password {{ '}}' }}"
+    inputs:
+      fields:
+        - type: "string"
+          id: "password"
+          label: "Password"
+          secret: true
+  register: custom_vault_cred_type
+
+- name: Create a credential of the custom type
+  awx.awx.credential:
+    credential_type: "{{ custom_vault_cred_type.id }}"
+    name: Credential From Vault
+    inputs: {}
+    organization: Default
+  register: custom_credential
+
+- name: Use the Vault Credential For the new credential
+  awx.awx.credential_input_source:
+    input_field_name: password
+    target_credential: "{{ custom_credential.id }}"
+    source_credential: "{{ vault_cred.id }}"
+    metadata:
+      auth_path: ""
+      secret_backend: "my_engine"
+      secret_key: "my_key"
+      secret_path: "/my_root/my_folder"
+      secret_version: ""

tools/docker-compose/ansible/roles/vault/tasks/unseal.yml

@@ -0,0 +1,14 @@
+---
+- name: Load vault keys
+  include_vars:
+    file: "{{ vault_file }}"
+
+- name: Unseal the vault
+  flowerysong.hvault.seal:
+    vault_addr: "http://localhost:1234"
+    state: unsealed
+    key: "{{ item }}"
+  loop:
+    - "{{ Unseal_Key_1 }}"
+    - "{{ Unseal_Key_2 }}"
+    - "{{ Unseal_Key_3 }}"

tools/docker-compose/ansible/unseal_vault.yml

@@ -0,0 +1,13 @@
+---
+- name: Run tasks post startup
+  hosts: localhost
+  gather_facts: False
+  tasks:
+    - name: Unseal the vault
+      include_role:
+        name: vault
+        tasks_from: unseal
+
+    - name: Display root token
+      debug:
+        var: Initial_Root_Token