From a949ee048a7b102dd926d22d101c6738b6d8506c Mon Sep 17 00:00:00 2001 From: Alan Rominger Date: Thu, 2 Feb 2023 15:27:18 -0500 Subject: [PATCH 1/4] Consolidate CI logic with github_ci_runner target Delete outright the step to install python Fix typo that failed to label stage --- .github/workflows/ci.yml | 56 ++++----------------- Makefile | 10 +++- tools/docker-compose/ansible/smoke-test.yml | 2 +- 3 files changed, 19 insertions(+), 49 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index faa2b02192..345d35563a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,8 +1,11 @@ --- name: CI env: - BRANCH: ${{ github.base_ref || 'devel' }} LC_ALL: "C.UTF-8" # prevent ERROR: Ansible could not initialize the preferred locale: unsupported locale setting + py_version: "3.9" + CI_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + DEV_DOCKER_TAG_BASE: ghcr.io/${{ github.repository_owner }} + COMPOSE_TAG: ${{ github.base_ref || 'devel' }} on: pull_request: jobs: @@ -43,60 +46,23 @@ jobs: steps: - uses: actions/checkout@v2 - - name: Get python version from Makefile - run: echo py_version=`make PYTHON_VERSION` >> $GITHUB_ENV + - name: Run check ${{ matrix.tests.label }} + run: make github_ci_runner + env: + CI_GITHUB_COMMAND: ${{ matrix.tests.command }} - - name: Install python ${{ env.py_version }} - uses: actions/setup-python@v2 - with: - python-version: ${{ env.py_version }} - - - name: Log in to registry - run: | - echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin - - - name: Pre-pull image to warm build cache - run: | - docker pull ghcr.io/${{ github.repository_owner }}/awx_devel:${{ env.BRANCH }} || : - - - name: Build image - run: | - DEV_DOCKER_TAG_BASE=ghcr.io/${{ github.repository_owner }} COMPOSE_TAG=${{ env.BRANCH }} make docker-compose-build - - - name: ${{ matrix.texts.label }} - run: | - docker run -u $(id -u) --rm -v ${{ github.workspace}}:/awx_devel/:Z \ - --workdir=/awx_devel ghcr.io/${{ github.repository_owner }}/awx_devel:${{ env.BRANCH }} ${{ matrix.tests.command }} dev-env: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - name: Get python version from Makefile - run: echo py_version=`make PYTHON_VERSION` >> $GITHUB_ENV - - name: Install python ${{ env.py_version }} uses: actions/setup-python@v2 with: python-version: ${{ env.py_version }} - - name: Log in to registry - run: | - echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin - - - name: Pre-pull image to warm build cache - run: | - docker pull ghcr.io/${{ github.repository_owner }}/awx_devel:${{ env.BRANCH }} || : - - - name: Build image - run: | - DEV_DOCKER_TAG_BASE=ghcr.io/${{ github.repository_owner }} COMPOSE_TAG=${{ env.BRANCH }} make docker-compose-build - - name: Run smoke test - run: | - export DEV_DOCKER_TAG_BASE=ghcr.io/${{ github.repository_owner }} - export COMPOSE_TAG=${{ env.BRANCH }} - ansible-playbook tools/docker-compose/ansible/smoke-test.yml -e repo_dir=$(pwd) -v + run: make github_ci_setup && ansible-playbook tools/docker-compose/ansible/smoke-test.yml -v awx-operator: runs-on: ubuntu-latest @@ -112,10 +78,6 @@ jobs: repository: ansible/awx-operator path: awx-operator - - name: Get python version from Makefile - working-directory: awx - run: echo py_version=`make PYTHON_VERSION` >> $GITHUB_ENV - - name: Install python ${{ env.py_version }} uses: actions/setup-python@v2 with: diff --git a/Makefile b/Makefile index 1185666e76..642513dace 100644 --- a/Makefile +++ b/Makefile @@ -65,7 +65,7 @@ I18N_FLAG_FILE = .i18n_built sdist \ ui-release ui-devel \ VERSION PYTHON_VERSION docker-compose-sources \ - .git/hooks/pre-commit + .git/hooks/pre-commit github_ci_setup github_ci_runner clean-tmp: rm -rf tmp/ @@ -301,6 +301,14 @@ test: cd awxkit && $(VENV_BASE)/awx/bin/tox -re py3 awx-manage check_migrations --dry-run --check -n 'missing_migration_file' +github_ci_setup: + echo $(CI_GITHUB_TOKEN) | docker login ghcr.io -u $(GITHUB_ACTOR) --password-stdin # Log in to registry + docker pull $(DEVEL_IMAGE_NAME) || : # Pre-pull image to warm build cache + make docker-compose-build # Build image + +github_ci_runner: github_ci_setup + docker run -u $(shell id -u) --rm -v $(GITHUB_WORKSPACE):/awx_devel/:Z --workdir=/awx_devel $(DEVEL_IMAGE_NAME) $(CI_GITHUB_COMMAND) + test_collection: rm -f $(shell ls -d $(VENV_BASE)/awx/lib/python* | head -n 1)/no-global-site-packages.txt if [ "$(VENV_BASE)" ]; then \ diff --git a/tools/docker-compose/ansible/smoke-test.yml b/tools/docker-compose/ansible/smoke-test.yml index 79b0fc2ee2..f4002c8567 100644 --- a/tools/docker-compose/ansible/smoke-test.yml +++ b/tools/docker-compose/ansible/smoke-test.yml @@ -17,7 +17,7 @@ environment: COMPOSE_UP_OPTS: -d args: - chdir: "{{ repo_dir }}" + chdir: "{{ playbook_dir }}/../../../" # Takes a while for migrations to finish - name: Wait for the dev environment to be ready From 59da9a29df135a91a9b61649c55dbfc60a4ade94 Mon Sep 17 00:00:00 2001 From: Alan Rominger Date: Thu, 2 Feb 2023 21:10:05 -0500 Subject: [PATCH 2/4] Delete everything about py_version in CI workflow --- .github/workflows/ci.yml | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 345d35563a..f492230244 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,7 +2,6 @@ name: CI env: LC_ALL: "C.UTF-8" # prevent ERROR: Ansible could not initialize the preferred locale: unsupported locale setting - py_version: "3.9" CI_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} DEV_DOCKER_TAG_BASE: ghcr.io/${{ github.repository_owner }} COMPOSE_TAG: ${{ github.base_ref || 'devel' }} @@ -56,11 +55,6 @@ jobs: steps: - uses: actions/checkout@v2 - - name: Install python ${{ env.py_version }} - uses: actions/setup-python@v2 - with: - python-version: ${{ env.py_version }} - - name: Run smoke test run: make github_ci_setup && ansible-playbook tools/docker-compose/ansible/smoke-test.yml -v @@ -78,6 +72,10 @@ jobs: repository: ansible/awx-operator path: awx-operator + - name: Get python version from Makefile + working-directory: awx + run: echo py_version=`make PYTHON_VERSION` >> $GITHUB_ENV + - name: Install python ${{ env.py_version }} uses: actions/setup-python@v2 with: From 0f3adb52b1a5345b93ed183e89fd22705db5f426 Mon Sep 17 00:00:00 2001 From: Alan Rominger Date: Fri, 3 Feb 2023 11:06:08 -0500 Subject: [PATCH 3/4] Add help comments and reorg targets for separation --- .github/workflows/ci.yml | 4 +--- Makefile | 15 +++++++++++---- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f492230244..dba6c5e6e7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -46,9 +46,7 @@ jobs: - uses: actions/checkout@v2 - name: Run check ${{ matrix.tests.label }} - run: make github_ci_runner - env: - CI_GITHUB_COMMAND: ${{ matrix.tests.command }} + run: AWX_DOCKER_CMD='${{ matrix.tests.command }}' make github_ci_runner dev-env: runs-on: ubuntu-latest diff --git a/Makefile b/Makefile index 642513dace..896dcf2c6c 100644 --- a/Makefile +++ b/Makefile @@ -301,13 +301,20 @@ test: cd awxkit && $(VENV_BASE)/awx/bin/tox -re py3 awx-manage check_migrations --dry-run --check -n 'missing_migration_file' +## Login to Github container image registry, pull image, then build image. github_ci_setup: - echo $(CI_GITHUB_TOKEN) | docker login ghcr.io -u $(GITHUB_ACTOR) --password-stdin # Log in to registry + # GITHUB_ACTOR is automatic github actions env var + # CI_GITHUB_TOKEN is defined in .github files + echo $(CI_GITHUB_TOKEN) | docker login ghcr.io -u $(GITHUB_ACTOR) --password-stdin docker pull $(DEVEL_IMAGE_NAME) || : # Pre-pull image to warm build cache - make docker-compose-build # Build image + make docker-compose-build -github_ci_runner: github_ci_setup - docker run -u $(shell id -u) --rm -v $(GITHUB_WORKSPACE):/awx_devel/:Z --workdir=/awx_devel $(DEVEL_IMAGE_NAME) $(CI_GITHUB_COMMAND) +## Runs AWX_DOCKER_CMD inside a new docker container. +docker-runner: + docker run -u $(shell id -u) --rm -v $(shell pwd):/awx_devel/:Z --workdir=/awx_devel $(DEVEL_IMAGE_NAME) $(AWX_DOCKER_CMD) + +## Builds image and runs AWX_DOCKER_CMD in it, mainly for .github checks. +github_ci_runner: github_ci_setup docker-runner test_collection: rm -f $(shell ls -d $(VENV_BASE)/awx/lib/python* | head -n 1)/no-global-site-packages.txt From 84822784e8e9992aac93f9397115d4408fc62c1e Mon Sep 17 00:00:00 2001 From: Alan Rominger Date: Fri, 3 Feb 2023 11:51:29 -0500 Subject: [PATCH 4/4] Get rid of label because it is confusing --- .github/workflows/ci.yml | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dba6c5e6e7..74d3f955f0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,32 +20,24 @@ jobs: tests: - name: api-test command: /start_tests.sh - label: Run API Tests - name: api-lint command: /var/lib/awx/venv/awx/bin/tox -e linters - label: Run API Linters - name: api-swagger command: /start_tests.sh swagger - label: Generate API Reference - name: awx-collection command: /start_tests.sh test_collection_all - label: Run Collection Tests - name: api-schema - label: Check API Schema command: /start_tests.sh detect-schema-change SCHEMA_DIFF_BASE_BRANCH=${{ github.event.pull_request.base.ref }} - name: ui-lint - label: Run UI Linters command: make ui-lint - name: ui-test-screens - label: Run UI Screens Tests command: make ui-test-screens - name: ui-test-general - label: Run UI General Tests command: make ui-test-general steps: - uses: actions/checkout@v2 - - name: Run check ${{ matrix.tests.label }} + - name: Run check ${{ matrix.tests.name }} run: AWX_DOCKER_CMD='${{ matrix.tests.command }}' make github_ci_runner dev-env: