Switch to ssh for private git requirements (#6838)

.github/actions/awx_devel_image/action.yml

@@ -4,8 +4,8 @@ inputs:
   github-token:
     description: GitHub Token for registry access
     required: true
-  private-github-token:
+  private-github-key:
-    description: GitHub Token for private repositories
+    description: GitHub private key for private repositories
     required: false
     default: ''
 runs:
@@ -26,10 +26,25 @@ runs:
       run: |
         echo "${{ inputs.github-token }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin
 
-    - name: Add Private github token to requirements_git.credentials.txt
+    - name: Generate placeholder SSH private key if SSH auth for private repos is not needed
+      id: generate_key
       shell: bash
-      run: echo "https://x-access-token:${{ inputs.private-github-token }}@github.com" >> requirements/requirements_git.credentials.txt
+      run: |
-      if: ${{ inputs.private-github-token != '' }}
+        if [[ -z "${{ inputs.private-github-key }}" ]]; then
+          ssh-keygen -t ed25519 -C "github-actions" -N "" -f ~/.ssh/id_ed25519
+          echo "SSH_PRIVATE_KEY<<EOF" >> $GITHUB_OUTPUT
+          cat ~/.ssh/id_ed25519 >> $GITHUB_OUTPUT
+          echo "EOF" >> $GITHUB_OUTPUT
+        else
+          echo "SSH_PRIVATE_KEY<<EOF" >> $GITHUB_OUTPUT
+          echo "${{ inputs.private-github-key }}" >> $GITHUB_OUTPUT
+          echo "EOF" >> $GITHUB_OUTPUT
+        fi
+
+    - name: Add private GitHub key to SSH agent
+      uses: webfactory/ssh-agent@v0.9.0
+      with:
+        ssh-private-key: ${{ steps.generate_key.outputs.SSH_PRIVATE_KEY }}
 
     - name: Pre-pull latest devel image to warm cache
       shell: bash
@@ -43,5 +58,5 @@ runs:
       shell: bash
       run: |
         DEV_DOCKER_TAG_BASE=ghcr.io/${OWNER_LC} \
-        COMPOSE_TAG=${{ github.base_ref }} \
+        COMPOSE_TAG=${{ github.base_ref || github.ref_name }} \
         make docker-compose-build

.github/actions/run_awx_devel/action.yml

@@ -9,8 +9,8 @@ inputs:
     required: false
     default: false
     type: boolean
-  private-github-token:
+  private-github-key:
-    description: GitHub Token for private repositories
+    description: GitHub private key for private repositories
     required: false
     default: ''
 outputs:
@@ -27,7 +27,7 @@ runs:
       uses: ./.github/actions/awx_devel_image
       with:
         github-token: ${{ inputs.github-token }}
-        private-github-token: ${{ inputs.private-github-token}}
+        private-github-key: ${{ inputs.private-github-key }}
 
     - name: Upgrade ansible-core
       shell: bash

.github/workflows/ci.yml

@@ -3,7 +3,6 @@ name: CI
 env:
   LC_ALL: "C.UTF-8" # prevent ERROR: Ansible could not initialize the preferred locale: unsupported locale setting
   CI_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-  CI_PRIVATE_GITHUB_TOKEN: ${{ secrets.PRIVATE_GITHUB_TOKEN }}
   DEV_DOCKER_OWNER: ${{ github.repository_owner }}
   COMPOSE_TAG: ${{ github.base_ref || 'devel' }}
 on:
@@ -47,7 +46,7 @@ jobs:
         uses: ./.github/actions/awx_devel_image
         with:
           github-token: ${{ secrets.GITHUB_TOKEN }}
-          private-github-token: ${{ secrets.PRIVATE_GITHUB_TOKEN }}
+          private-github-key: ${{ secrets.PRIVATE_GITHUB_KEY }}
 
       - name: Run check ${{ matrix.tests.name }}
         run: AWX_DOCKER_CMD='${{ matrix.tests.command }}' make docker-runner
@@ -69,7 +68,7 @@ jobs:
         with:
           build-ui: false
           github-token: ${{ secrets.GITHUB_TOKEN }}
-          private-github-token: ${{ secrets.PRIVATE_GITHUB_TOKEN }}
+          private-github-key: ${{ secrets.PRIVATE_GITHUB_KEY }}
 
       - name: Run smoke test
         run: ansible-playbook tools/docker-compose/ansible/smoke-test.yml -v
@@ -106,11 +105,25 @@ jobs:
         run: |
           python3 -m pip install docker
 
-      - name: Add Private github token to requirements_git.credentials.txt
+      - name: Generate placeholder SSH private key if SSH auth for private repos is not needed
+        id: generate_key
         shell: bash
-        working-directory: awx
+        run: |
-        run: echo "https://x-access-token:${{ env.CI_PRIVATE_GITHUB_TOKEN }}@github.com" >> requirements/requirements_git.credentials.txt
+          if [[ -z "${{ secrets.PRIVATE_GITHUB_KEY }}" ]]; then
-        if: ${{ env.CI_PRIVATE_GITHUB_TOKEN != '' }}
+            ssh-keygen -t ed25519 -C "github-actions" -N "" -f ~/.ssh/id_ed25519
+            echo "SSH_PRIVATE_KEY<<EOF" >> $GITHUB_OUTPUT
+            cat ~/.ssh/id_ed25519 >> $GITHUB_OUTPUT
+            echo "EOF" >> $GITHUB_OUTPUT
+          else
+            echo "SSH_PRIVATE_KEY<<EOF" >> $GITHUB_OUTPUT
+            echo "${{ secrets.PRIVATE_GITHUB_KEY }}" >> $GITHUB_OUTPUT
+            echo "EOF" >> $GITHUB_OUTPUT
+          fi
+
+      - name: Add private GitHub key to SSH agent
+        uses: webfactory/ssh-agent@v0.9.0
+        with:
+          ssh-private-key: ${{ steps.generate_key.outputs.SSH_PRIVATE_KEY }}
 
       - name: Build AWX image
         working-directory: awx
@@ -217,7 +230,7 @@ jobs:
         with:
           build-ui: false
           github-token: ${{ secrets.GITHUB_TOKEN }}
-          private-github-token: ${{ secrets.PRIVATE_GITHUB_TOKEN }}
+          private-github-key: ${{ secrets.PRIVATE_GITHUB_KEY }}
 
       - name: Install dependencies for running tests
         run: |

.github/workflows/devel_images.yml

@@ -3,7 +3,6 @@ name: Build/Push Development Images
 env:
   LC_ALL: "C.UTF-8" # prevent ERROR: Ansible could not initialize the preferred locale: unsupported locale setting
   DOCKER_CACHE: "--no-cache" # using the cache will not rebuild git requirements and other things
-  CI_PRIVATE_GITHUB_TOKEN: ${{ secrets.PRIVATE_GITHUB_TOKEN }}
 on:
   workflow_dispatch:
   push:
@@ -86,10 +85,25 @@ jobs:
           make ui-next
         if: matrix.build-targets.image-name == 'awx'
 
-      - name: Add private GitHub token to requirements_git.credentials.txt
+      - name: Generate placeholder SSH private key if SSH auth for private repos is not needed
+        id: generate_key
         shell: bash
-        run: echo "https://x-access-token:${{ secrets.PRIVATE_GITHUB_TOKEN }}@github.com" >> requirements/requirements_git.credentials.txt
+        run: |
-        if: ${{ env.CI_PRIVATE_GITHUB_TOKEN != '' }}
+          if [[ -z "${{ secrets.PRIVATE_GITHUB_KEY }}" ]]; then
+            ssh-keygen -t ed25519 -C "github-actions" -N "" -f ~/.ssh/id_ed25519
+            echo "SSH_PRIVATE_KEY<<EOF" >> $GITHUB_OUTPUT
+            cat ~/.ssh/id_ed25519 >> $GITHUB_OUTPUT
+            echo "EOF" >> $GITHUB_OUTPUT
+          else
+            echo "SSH_PRIVATE_KEY<<EOF" >> $GITHUB_OUTPUT
+            echo "${{ secrets.PRIVATE_GITHUB_KEY }}" >> $GITHUB_OUTPUT
+            echo "EOF" >> $GITHUB_OUTPUT
+          fi
+
+      - name: Add private GitHub key to SSH agent
+        uses: webfactory/ssh-agent@v0.9.0
+        with:
+          ssh-private-key: ${{ steps.generate_key.outputs.SSH_PRIVATE_KEY }}
 
       - name: Build and push AWX devel images
         run: |

Makefile

@@ -618,6 +618,7 @@ Dockerfile.dev: tools/ansible/roles/dockerfile/templates/Dockerfile.j2
 ## Build awx_devel image for docker compose development environment
 docker-compose-build: Dockerfile.dev
 	DOCKER_BUILDKIT=1 docker build \
+		--ssh default=$(SSH_AUTH_SOCK) \
 		-f Dockerfile.dev \
 		-t $(DEVEL_IMAGE_NAME) \
 		--build-arg BUILDKIT_INLINE_CACHE=1 \
@@ -629,6 +630,7 @@ docker-compose-buildx: Dockerfile.dev
 	- docker buildx create --name docker-compose-buildx
 	docker buildx use docker-compose-buildx
 	- docker buildx build \
+		--ssh default=$(SSH_AUTH_SOCK) \
 		--push \
 		--build-arg BUILDKIT_INLINE_CACHE=1 \
 		$(DOCKER_DEVEL_CACHE_FLAG) \
@@ -691,6 +693,7 @@ Dockerfile: tools/ansible/roles/dockerfile/templates/Dockerfile.j2
 ## Build awx image for deployment on Kubernetes environment.
 awx-kube-build: Dockerfile
 	DOCKER_BUILDKIT=1 docker build -f Dockerfile \
+		--ssh default=$(SSH_AUTH_SOCK) \
 		--build-arg VERSION=$(VERSION) \
 		--build-arg SETUPTOOLS_SCM_PRETEND_VERSION=$(VERSION) \
 		--build-arg HEADLESS=$(HEADLESS) \
@@ -702,6 +705,7 @@ awx-kube-buildx: Dockerfile
 	- docker buildx create --name awx-kube-buildx
 	docker buildx use awx-kube-buildx
 	- docker buildx build \
+		--ssh default=$(SSH_AUTH_SOCK) \
 		--push \
 		--build-arg VERSION=$(VERSION) \
 		--build-arg SETUPTOOLS_SCM_PRETEND_VERSION=$(VERSION) \
@@ -725,6 +729,7 @@ Dockerfile.kube-dev: tools/ansible/roles/dockerfile/templates/Dockerfile.j2
 ## Build awx_kube_devel image for development on local Kubernetes environment.
 awx-kube-dev-build: Dockerfile.kube-dev
 	DOCKER_BUILDKIT=1 docker build -f Dockerfile.kube-dev \
+		--ssh default=$(SSH_AUTH_SOCK) \
 	    --build-arg BUILDKIT_INLINE_CACHE=1 \
 	     $(DOCKER_KUBE_DEV_CACHE_FLAG) \
 	    -t $(IMAGE_KUBE_DEV) .
@@ -734,6 +739,7 @@ awx-kube-dev-buildx: Dockerfile.kube-dev
 	- docker buildx create --name awx-kube-dev-buildx
 	docker buildx use awx-kube-dev-buildx
 	- docker buildx build \
+		--ssh default=$(SSH_AUTH_SOCK) \
 		--push \
 		--build-arg BUILDKIT_INLINE_CACHE=1 \
 		$(DOCKER_KUBE_DEV_CACHE_FLAG) \

requirements/requirements_git.txt

@@ -1,4 +1,4 @@
 git+https://github.com/ansible/system-certifi.git@devel#egg=certifi
 # Remove pbr from requirements.in when moving ansible-runner to requirements.in
 git+https://github.com/ansible/python3-saml.git@devel#egg=python3-saml
-django-ansible-base @ git+https://github.com/ansible-automation-platform/django-ansible-base@stable-2.5#egg=django-ansible-base[rest-filters,jwt_consumer,resource-registry,rbac,feature-flags]
+django-ansible-base @ git+ssh://git@github.com/ansible-automation-platform/django-ansible-base@stable-2.5#egg=django-ansible-base[rest-filters,jwt_consumer,resource-registry,rbac,feature-flags]

tools/ansible/roles/dockerfile/templates/Dockerfile.j2

@@ -22,6 +22,7 @@ RUN rpm --import /etc/pki/rpm-gpg/RPM-GPG-KEY-centosofficial
 RUN dnf -y update && dnf install -y 'dnf-command(config-manager)' && \
     dnf config-manager --set-enabled crb && \
     dnf -y install \
+    openssh-clients \
     iputils \
     gcc \
     gcc-c++ \
@@ -50,6 +51,10 @@ RUN dnf -y update && dnf install -y 'dnf-command(config-manager)' && \
     xmlsec1-devel \
     xmlsec1-openssl-devel
 
+# Add github.com to known hosts
+RUN mkdir -p ~/.ssh && chmod 0700 ~/.ssh
+RUN ssh-keyscan github.com > ~/.ssh/known_hosts
+
 RUN pip3.11 install -vv build
 
 {% if image_architecture == 'ppc64le'  %}
@@ -70,7 +75,7 @@ ADD requirements/requirements.txt \
     /tmp/requirements/
 
 RUN git config --global credential.helper "store --file=/tmp/requirements/requirements_git.credentials.txt"
-RUN cd /tmp && make requirements_awx
+RUN --mount=type=ssh cd /tmp && make requirements_awx
 
 ARG VERSION
 ARG SETUPTOOLS_SCM_PRETEND_VERSION