diff --git a/.github/actions/awx_devel_image/action.yml b/.github/actions/awx_devel_image/action.yml index 96cda9288a..142c73b198 100644 --- a/.github/actions/awx_devel_image/action.yml +++ b/.github/actions/awx_devel_image/action.yml @@ -4,8 +4,8 @@ inputs: github-token: description: GitHub Token for registry access required: true - private-github-token: - description: GitHub Token for private repositories + private-github-key: + description: GitHub private key for private repositories required: false default: '' runs: @@ -26,10 +26,25 @@ runs: run: | echo "${{ inputs.github-token }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin - - name: Add Private github token to requirements_git.credentials.txt + - name: Generate placeholder SSH private key if SSH auth for private repos is not needed + id: generate_key shell: bash - run: echo "https://x-access-token:${{ inputs.private-github-token }}@github.com" >> requirements/requirements_git.credentials.txt - if: ${{ inputs.private-github-token != '' }} + run: | + if [[ -z "${{ inputs.private-github-key }}" ]]; then + ssh-keygen -t ed25519 -C "github-actions" -N "" -f ~/.ssh/id_ed25519 + echo "SSH_PRIVATE_KEY<> $GITHUB_OUTPUT + cat ~/.ssh/id_ed25519 >> $GITHUB_OUTPUT + echo "EOF" >> $GITHUB_OUTPUT + else + echo "SSH_PRIVATE_KEY<> $GITHUB_OUTPUT + echo "${{ inputs.private-github-key }}" >> $GITHUB_OUTPUT + echo "EOF" >> $GITHUB_OUTPUT + fi + + - name: Add private GitHub key to SSH agent + uses: webfactory/ssh-agent@v0.9.0 + with: + ssh-private-key: ${{ steps.generate_key.outputs.SSH_PRIVATE_KEY }} - name: Pre-pull latest devel image to warm cache shell: bash @@ -43,5 +58,5 @@ runs: shell: bash run: | DEV_DOCKER_TAG_BASE=ghcr.io/${OWNER_LC} \ - COMPOSE_TAG=${{ github.base_ref }} \ + COMPOSE_TAG=${{ github.base_ref || github.ref_name }} \ make docker-compose-build diff --git a/.github/actions/run_awx_devel/action.yml b/.github/actions/run_awx_devel/action.yml index 3fbaeb88c0..62eb920771 100644 --- a/.github/actions/run_awx_devel/action.yml +++ b/.github/actions/run_awx_devel/action.yml @@ -9,8 +9,8 @@ inputs: required: false default: false type: boolean - private-github-token: - description: GitHub Token for private repositories + private-github-key: + description: GitHub private key for private repositories required: false default: '' outputs: @@ -27,7 +27,7 @@ runs: uses: ./.github/actions/awx_devel_image with: github-token: ${{ inputs.github-token }} - private-github-token: ${{ inputs.private-github-token}} + private-github-key: ${{ inputs.private-github-key }} - name: Upgrade ansible-core shell: bash diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c8eb80c9f6..27bd575b4e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -3,7 +3,6 @@ name: CI env: LC_ALL: "C.UTF-8" # prevent ERROR: Ansible could not initialize the preferred locale: unsupported locale setting CI_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - CI_PRIVATE_GITHUB_TOKEN: ${{ secrets.PRIVATE_GITHUB_TOKEN }} DEV_DOCKER_OWNER: ${{ github.repository_owner }} COMPOSE_TAG: ${{ github.base_ref || 'devel' }} on: @@ -47,7 +46,7 @@ jobs: uses: ./.github/actions/awx_devel_image with: github-token: ${{ secrets.GITHUB_TOKEN }} - private-github-token: ${{ secrets.PRIVATE_GITHUB_TOKEN }} + private-github-key: ${{ secrets.PRIVATE_GITHUB_KEY }} - name: Run check ${{ matrix.tests.name }} run: AWX_DOCKER_CMD='${{ matrix.tests.command }}' make docker-runner @@ -69,7 +68,7 @@ jobs: with: build-ui: false github-token: ${{ secrets.GITHUB_TOKEN }} - private-github-token: ${{ secrets.PRIVATE_GITHUB_TOKEN }} + private-github-key: ${{ secrets.PRIVATE_GITHUB_KEY }} - name: Run smoke test run: ansible-playbook tools/docker-compose/ansible/smoke-test.yml -v @@ -106,11 +105,25 @@ jobs: run: | python3 -m pip install docker - - name: Add Private github token to requirements_git.credentials.txt + - name: Generate placeholder SSH private key if SSH auth for private repos is not needed + id: generate_key shell: bash - working-directory: awx - run: echo "https://x-access-token:${{ env.CI_PRIVATE_GITHUB_TOKEN }}@github.com" >> requirements/requirements_git.credentials.txt - if: ${{ env.CI_PRIVATE_GITHUB_TOKEN != '' }} + run: | + if [[ -z "${{ secrets.PRIVATE_GITHUB_KEY }}" ]]; then + ssh-keygen -t ed25519 -C "github-actions" -N "" -f ~/.ssh/id_ed25519 + echo "SSH_PRIVATE_KEY<> $GITHUB_OUTPUT + cat ~/.ssh/id_ed25519 >> $GITHUB_OUTPUT + echo "EOF" >> $GITHUB_OUTPUT + else + echo "SSH_PRIVATE_KEY<> $GITHUB_OUTPUT + echo "${{ secrets.PRIVATE_GITHUB_KEY }}" >> $GITHUB_OUTPUT + echo "EOF" >> $GITHUB_OUTPUT + fi + + - name: Add private GitHub key to SSH agent + uses: webfactory/ssh-agent@v0.9.0 + with: + ssh-private-key: ${{ steps.generate_key.outputs.SSH_PRIVATE_KEY }} - name: Build AWX image working-directory: awx @@ -217,7 +230,7 @@ jobs: with: build-ui: false github-token: ${{ secrets.GITHUB_TOKEN }} - private-github-token: ${{ secrets.PRIVATE_GITHUB_TOKEN }} + private-github-key: ${{ secrets.PRIVATE_GITHUB_KEY }} - name: Install dependencies for running tests run: | diff --git a/.github/workflows/devel_images.yml b/.github/workflows/devel_images.yml index 17e2752687..b29d236fd6 100644 --- a/.github/workflows/devel_images.yml +++ b/.github/workflows/devel_images.yml @@ -3,7 +3,6 @@ name: Build/Push Development Images env: LC_ALL: "C.UTF-8" # prevent ERROR: Ansible could not initialize the preferred locale: unsupported locale setting DOCKER_CACHE: "--no-cache" # using the cache will not rebuild git requirements and other things - CI_PRIVATE_GITHUB_TOKEN: ${{ secrets.PRIVATE_GITHUB_TOKEN }} on: workflow_dispatch: push: @@ -86,10 +85,25 @@ jobs: make ui-next if: matrix.build-targets.image-name == 'awx' - - name: Add private GitHub token to requirements_git.credentials.txt + - name: Generate placeholder SSH private key if SSH auth for private repos is not needed + id: generate_key shell: bash - run: echo "https://x-access-token:${{ secrets.PRIVATE_GITHUB_TOKEN }}@github.com" >> requirements/requirements_git.credentials.txt - if: ${{ env.CI_PRIVATE_GITHUB_TOKEN != '' }} + run: | + if [[ -z "${{ secrets.PRIVATE_GITHUB_KEY }}" ]]; then + ssh-keygen -t ed25519 -C "github-actions" -N "" -f ~/.ssh/id_ed25519 + echo "SSH_PRIVATE_KEY<> $GITHUB_OUTPUT + cat ~/.ssh/id_ed25519 >> $GITHUB_OUTPUT + echo "EOF" >> $GITHUB_OUTPUT + else + echo "SSH_PRIVATE_KEY<> $GITHUB_OUTPUT + echo "${{ secrets.PRIVATE_GITHUB_KEY }}" >> $GITHUB_OUTPUT + echo "EOF" >> $GITHUB_OUTPUT + fi + + - name: Add private GitHub key to SSH agent + uses: webfactory/ssh-agent@v0.9.0 + with: + ssh-private-key: ${{ steps.generate_key.outputs.SSH_PRIVATE_KEY }} - name: Build and push AWX devel images run: | diff --git a/Makefile b/Makefile index bab5d27c95..68b2c66c38 100644 --- a/Makefile +++ b/Makefile @@ -618,6 +618,7 @@ Dockerfile.dev: tools/ansible/roles/dockerfile/templates/Dockerfile.j2 ## Build awx_devel image for docker compose development environment docker-compose-build: Dockerfile.dev DOCKER_BUILDKIT=1 docker build \ + --ssh default=$(SSH_AUTH_SOCK) \ -f Dockerfile.dev \ -t $(DEVEL_IMAGE_NAME) \ --build-arg BUILDKIT_INLINE_CACHE=1 \ @@ -629,6 +630,7 @@ docker-compose-buildx: Dockerfile.dev - docker buildx create --name docker-compose-buildx docker buildx use docker-compose-buildx - docker buildx build \ + --ssh default=$(SSH_AUTH_SOCK) \ --push \ --build-arg BUILDKIT_INLINE_CACHE=1 \ $(DOCKER_DEVEL_CACHE_FLAG) \ @@ -691,6 +693,7 @@ Dockerfile: tools/ansible/roles/dockerfile/templates/Dockerfile.j2 ## Build awx image for deployment on Kubernetes environment. awx-kube-build: Dockerfile DOCKER_BUILDKIT=1 docker build -f Dockerfile \ + --ssh default=$(SSH_AUTH_SOCK) \ --build-arg VERSION=$(VERSION) \ --build-arg SETUPTOOLS_SCM_PRETEND_VERSION=$(VERSION) \ --build-arg HEADLESS=$(HEADLESS) \ @@ -702,6 +705,7 @@ awx-kube-buildx: Dockerfile - docker buildx create --name awx-kube-buildx docker buildx use awx-kube-buildx - docker buildx build \ + --ssh default=$(SSH_AUTH_SOCK) \ --push \ --build-arg VERSION=$(VERSION) \ --build-arg SETUPTOOLS_SCM_PRETEND_VERSION=$(VERSION) \ @@ -725,6 +729,7 @@ Dockerfile.kube-dev: tools/ansible/roles/dockerfile/templates/Dockerfile.j2 ## Build awx_kube_devel image for development on local Kubernetes environment. awx-kube-dev-build: Dockerfile.kube-dev DOCKER_BUILDKIT=1 docker build -f Dockerfile.kube-dev \ + --ssh default=$(SSH_AUTH_SOCK) \ --build-arg BUILDKIT_INLINE_CACHE=1 \ $(DOCKER_KUBE_DEV_CACHE_FLAG) \ -t $(IMAGE_KUBE_DEV) . @@ -734,6 +739,7 @@ awx-kube-dev-buildx: Dockerfile.kube-dev - docker buildx create --name awx-kube-dev-buildx docker buildx use awx-kube-dev-buildx - docker buildx build \ + --ssh default=$(SSH_AUTH_SOCK) \ --push \ --build-arg BUILDKIT_INLINE_CACHE=1 \ $(DOCKER_KUBE_DEV_CACHE_FLAG) \ diff --git a/requirements/requirements_git.txt b/requirements/requirements_git.txt index f6c4cc00b8..2c6e82524e 100644 --- a/requirements/requirements_git.txt +++ b/requirements/requirements_git.txt @@ -1,4 +1,4 @@ git+https://github.com/ansible/system-certifi.git@devel#egg=certifi # Remove pbr from requirements.in when moving ansible-runner to requirements.in git+https://github.com/ansible/python3-saml.git@devel#egg=python3-saml -django-ansible-base @ git+https://github.com/ansible-automation-platform/django-ansible-base@stable-2.5#egg=django-ansible-base[rest-filters,jwt_consumer,resource-registry,rbac,feature-flags] +django-ansible-base @ git+ssh://git@github.com/ansible-automation-platform/django-ansible-base@stable-2.5#egg=django-ansible-base[rest-filters,jwt_consumer,resource-registry,rbac,feature-flags] diff --git a/tools/ansible/roles/dockerfile/templates/Dockerfile.j2 b/tools/ansible/roles/dockerfile/templates/Dockerfile.j2 index d887002023..356dffd1fc 100644 --- a/tools/ansible/roles/dockerfile/templates/Dockerfile.j2 +++ b/tools/ansible/roles/dockerfile/templates/Dockerfile.j2 @@ -22,6 +22,7 @@ RUN rpm --import /etc/pki/rpm-gpg/RPM-GPG-KEY-centosofficial RUN dnf -y update && dnf install -y 'dnf-command(config-manager)' && \ dnf config-manager --set-enabled crb && \ dnf -y install \ + openssh-clients \ iputils \ gcc \ gcc-c++ \ @@ -50,6 +51,10 @@ RUN dnf -y update && dnf install -y 'dnf-command(config-manager)' && \ xmlsec1-devel \ xmlsec1-openssl-devel +# Add github.com to known hosts +RUN mkdir -p ~/.ssh && chmod 0700 ~/.ssh +RUN ssh-keyscan github.com > ~/.ssh/known_hosts + RUN pip3.11 install -vv build {% if image_architecture == 'ppc64le' %} @@ -70,7 +75,7 @@ ADD requirements/requirements.txt \ /tmp/requirements/ RUN git config --global credential.helper "store --file=/tmp/requirements/requirements_git.credentials.txt" -RUN cd /tmp && make requirements_awx +RUN --mount=type=ssh cd /tmp && make requirements_awx ARG VERSION ARG SETUPTOOLS_SCM_PRETEND_VERSION