Add hashivault option as docker-compose optional container (#14161)

Co-authored-by: Sarabraj Singh <singh.sarabraj@gmail.com>

Makefile

@@ -37,6 +37,8 @@ SPLUNK ?= false
 PROMETHEUS ?= false
 # If set to true docker-compose will also start a grafana instance
 GRAFANA ?= false
+# If set to true docker-compose will also start a hashicorp vault instance
+VAULT ?= false
 # If set to true docker-compose will also start a tacacs+ instance
 TACACS ?= false
 
@@ -525,6 +527,7 @@ docker-compose-sources: .git/hooks/pre-commit
 	    -e enable_splunk=$(SPLUNK) \
 	    -e enable_prometheus=$(PROMETHEUS) \
 	    -e enable_grafana=$(GRAFANA) \
+	    -e enable_vault=$(VAULT) \
 	    -e enable_tacacs=$(TACACS) \
             $(EXTRA_SOURCES_ANSIBLE_OPTS)
 

awx/main/credential_plugins/hashivault.py

@@ -265,6 +265,8 @@ def kv_backend(**kwargs):
 
     if secret_key:
         try:
+            if (secret_key != 'data') and (secret_key not in json['data']) and ('data' in json['data']):
+                return json['data']['data'][secret_key]
             return json['data'][secret_key]
         except KeyError:
             raise RuntimeError('{} is not present at {}'.format(secret_key, secret_path))

tools/docker-compose/README.md

@@ -303,7 +303,7 @@ To bring up a 1 node AWX + minikube that is accessible from AWX run the followin
 Start minikube
 
 ```bash
-(host)$minikube start --cpus=4  --memory=8g --addons=ingress`
+(host)$minikube start --cpus=4  --memory=8g --addons=ingress
 ```
 
 Start AWX
@@ -497,6 +497,49 @@ ansible-playbook tools/docker-compose/ansible/plumb_tacacs.yml
 
 Once the playbook is done running tacacs+ should now be setup in your development environment. This server has the accounts listed on https://hub.docker.com/r/dchidell/docker-tacacs
 
+### HashiVault Integration
+
+Run a HashiVault container alongside of AWX.
+
+```bash
+VAULT=true make docker-compose
+```
+
+Go to `http://localhost:1234` sign in with method "Token".
+
+You can find the generated token at `tools/docker-compose/_sources/secrets/vault_password.yml`,
+this is a root token, and it should not need a corresponding username.
+Note that the token will be different on each restart, as it is re-generated by the playbook,
+and the container does not use a persistent volume.
+
+As a demo, click "Enable new engine +", click "KV" and Next.
+In the "Path" enter "my_engine" and click "Enable Engine".
+Click on the name of the engine and then "Create secret +".
+In the "Path for this secret" enter "my_root/my_folder" and in the "Secret Data" put "my_key" for key and **"my_value"** for value.
+
+Then go to AWX and create a new HashiVault credential with the generated token.
+Then go to any other arbitrary credential and click the key icon on an input to use a credential lookup plugin.
+In the "External Secret Management System" menu, first select the already-created HashiVault credential.
+
+Then in the "Metadata" menu, put in this data which is important for the integration:
+ - Name of Secret Backend: "my_engine"
+ - Path to Secret: "data/my_root/my_folder"
+ - Key Name: "my_key"
+
+After this, apply the credential to a job template that writes the data in a debug task.
+In the job output, you should see **my_value**.
+
+(NOTE: the "arbitrary credential" could be a new custom credential type that injects to extra vars
+which is used in corresponding playbook that prints hostvars, but this doc assumes you know how to do that)
+
+The extremely non-obvious input is the fact that the fact prefixes "data/" unexpectedly.
+This was discovered by inspecting the secret with the vault CLI, which may help with future troubleshooting.
+
+```
+docker exec -it -e VAULT_TOKEN=<token> tools_vault_1 vault kv get --address=http://127.0.0.1:1234 my_engine/my_root/my_folder
+```
+
+
 ### Prometheus and Grafana integration
 
 See docs at https://github.com/ansible/awx/blob/devel/tools/grafana/README.md

tools/docker-compose/ansible/roles/sources/tasks/main.yml

@@ -101,6 +101,10 @@
   include_tasks: ldap.yml
   when: enable_ldap | bool
 
+- name: Include Vault tasks if enabled
+  include_tasks: vault.yaml
+  when: enable_vault | bool
+
 - name: Render Docker-Compose
   template:
     src: docker-compose.yml.j2

tools/docker-compose/ansible/roles/sources/tasks/vault.yaml

@@ -0,0 +1,20 @@
+---
+- name: create vault secret file and scope into ansible-runtime
+  block:
+    - ansible.builtin.stat:
+        path: "{{ sources_dest }}/secrets/{{ item }}.yml"
+      register: vault_secret
+      loop:
+        - vault_password
+
+    - ansible.builtin.template:
+        src: "secrets.yml.j2"
+        dest: "{{ sources_dest }}/secrets/{{ item.item }}.yml"
+        mode: "0600"
+      loop: "{{ vault_secret.results }}"
+      loop_control:
+        label: "{{ item.item }}"
+
+    - include_vars: "{{ sources_dest }}/secrets/{{ item.item }}.yml"
+      loop: "{{ vault_secret.results }}"
+      no_log: true

tools/docker-compose/ansible/roles/sources/templates/docker-compose.yml.j2

@@ -233,6 +233,19 @@ services:
     privileged: true
   {% endfor %}
 {% endif %}
+{% if enable_vault|bool %}
+  vault:
+    image: hashicorp/vault:latest
+    container_name: tools_vault_1
+    hostname: vault
+    ports:
+      - "1234:1234"
+    environment:
+      VAULT_DEV_ROOT_TOKEN_ID: "{{ vault_password }}"
+      VAULT_DEV_LISTEN_ADDRESS: "0.0.0.0:1234"
+    cap_add:
+      - IPC_LOCK
+{% endif %}
 
 volumes:
   awx_db: