From c8b906ffb7aeeeeabb5c3e79484efc6d72858640 Mon Sep 17 00:00:00 2001 From: John Westcott IV <32551173+john-westcott-iv@users.noreply.github.com> Date: Mon, 7 Feb 2022 15:42:35 -0500 Subject: [PATCH] Workflow changes (#11692) Modifying workflows to install python for make commands Squashing CI tasks to remove repeated steps Modifying pre-commit.sh to not fail if there are no python file changes --- .github/workflows/ci.yml | 183 ++++++---------------------- .github/workflows/devel_image.yml | 8 ++ .github/workflows/e2e_test.yml | 8 ++ .github/workflows/stage.yml | 8 ++ .github/workflows/upload_schema.yml | 8 ++ Makefile | 8 +- pre-commit.sh | 12 +- 7 files changed, 81 insertions(+), 154 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2fa6fd3029..796d66e679 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,14 +5,48 @@ env: on: pull_request: jobs: - api-test: + common_tests: + name: ${{ matrix.tests.name }} runs-on: ubuntu-latest permissions: packages: write contents: read + strategy: + fail-fast: false + matrix: + tests: + - name: api-test + command: /start_tests.sh + label: Run API Tests + - name: api-lint + command: /var/lib/awx/venv/awx/bin/tox -e linters + label: Run API Linters + - name: api-swagger + command: /start_tests.sh swagger + label: Generate API Reference + - name: awx-collection + command: /start_tests.sh test_collection_all + label: Run Collection Tests + - name: api-schema + label: Check API Schema + command: /start_tests.sh detect-schema-change + - name: ui-lint + label: Run UI Linters + command: make ui-lint + - name: ui-test + label: Run UI Tests + command: make ui-test steps: - uses: actions/checkout@v2 + - name: Get python version from Makefile + run: echo py_version=`make PYTHON_VERSION` >> $GITHUB_ENV + + - name: Install python ${{ env.py_version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ env.py_version }} + - name: Log in to registry run: | echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin @@ -25,154 +59,11 @@ jobs: run: | DEV_DOCKER_TAG_BASE=ghcr.io/${{ github.repository_owner }} COMPOSE_TAG=${{ env.BRANCH }} make docker-compose-build - - name: Run API Tests + - name: ${{ matrix.texts.label }} run: | docker run -u $(id -u) --rm -v ${{ github.workspace}}:/awx_devel/:Z \ - --workdir=/awx_devel ghcr.io/${{ github.repository_owner }}/awx_devel:${{ env.BRANCH }} /start_tests.sh - api-lint: - runs-on: ubuntu-latest - permissions: - packages: write - contents: read - steps: - - uses: actions/checkout@v2 + --workdir=/awx_devel ghcr.io/${{ github.repository_owner }}/awx_devel:${{ env.BRANCH }} ${{ matrix.tests.command }} - - name: Log in to registry - run: | - echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin - - - name: Pre-pull image to warm build cache - run: | - docker pull ghcr.io/${{ github.repository_owner }}/awx_devel:${{ env.BRANCH }} || : - - - name: Build image - run: | - DEV_DOCKER_TAG_BASE=ghcr.io/${{ github.repository_owner }} COMPOSE_TAG=${{ env.BRANCH }} make docker-compose-build - - - name: Run API Linters - run: | - docker run -u $(id -u) --rm -v ${{ github.workspace}}:/awx_devel/:Z \ - --workdir=/awx_devel ghcr.io/${{ github.repository_owner }}/awx_devel:${{ env.BRANCH }} /var/lib/awx/venv/awx/bin/tox -e linters - api-swagger: - runs-on: ubuntu-latest - permissions: - packages: write - contents: read - steps: - - uses: actions/checkout@v2 - - - name: Log in to registry - run: | - echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin - - - name: Pre-pull image to warm build cache - run: | - docker pull ghcr.io/${{ github.repository_owner }}/awx_devel:${{ env.BRANCH }} || : || : - - - name: Build image - run: | - DEV_DOCKER_TAG_BASE=ghcr.io/${{ github.repository_owner }} COMPOSE_TAG=${{ env.BRANCH }} make docker-compose-build - - - name: Generate API Reference - run: | - docker run -u $(id -u) --rm -v ${{ github.workspace}}:/awx_devel/:Z \ - --workdir=/awx_devel ghcr.io/${{ github.repository_owner }}/awx_devel:${{ env.BRANCH }} /start_tests.sh swagger - awx-collection: - runs-on: ubuntu-latest - permissions: - packages: write - contents: read - steps: - - uses: actions/checkout@v2 - - - name: Log in to registry - run: | - echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin - - - name: Pre-pull image to warm build cache - run: | - docker pull ghcr.io/${{ github.repository_owner }}/awx_devel:${{ env.BRANCH }} || : - - - name: Build image - run: | - DEV_DOCKER_TAG_BASE=ghcr.io/${{ github.repository_owner }} COMPOSE_TAG=${{ env.BRANCH }} make docker-compose-build - - - name: Run Collection Tests - run: | - docker run -u $(id -u) --rm -v ${{ github.workspace}}:/awx_devel/:Z \ - --workdir=/awx_devel ghcr.io/${{ github.repository_owner }}/awx_devel:${{ env.BRANCH }} /start_tests.sh test_collection_all - api-schema: - runs-on: ubuntu-latest - permissions: - packages: write - contents: read - steps: - - uses: actions/checkout@v2 - - - name: Log in to registry - run: | - echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin - - - name: Pre-pull image to warm build cache - run: | - docker pull ghcr.io/${{ github.repository_owner }}/awx_devel:${{ env.BRANCH }} || : - - - name: Build image - run: | - DEV_DOCKER_TAG_BASE=ghcr.io/${{ github.repository_owner }} COMPOSE_TAG=${{ env.BRANCH }} make docker-compose-build - - - name: Check API Schema - run: | - docker run -u $(id -u) --rm -v ${{ github.workspace}}:/awx_devel/:Z \ - --workdir=/awx_devel ghcr.io/${{ github.repository_owner }}/awx_devel:${{ env.BRANCH }} /start_tests.sh detect-schema-change - ui-lint: - runs-on: ubuntu-latest - permissions: - packages: write - contents: read - steps: - - uses: actions/checkout@v2 - - - name: Log in to registry - run: | - echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin - - - name: Pre-pull image to warm build cache - run: | - docker pull ghcr.io/${{ github.repository_owner }}/awx_devel:${{ env.BRANCH }} || : - - - name: Build image - run: | - DEV_DOCKER_TAG_BASE=ghcr.io/${{ github.repository_owner }} COMPOSE_TAG=${{ env.BRANCH }} make docker-compose-build - - - name: Run UI Linters - run: | - docker run -u $(id -u) --rm -v ${{ github.workspace}}:/awx_devel/:Z \ - --workdir=/awx_devel ghcr.io/${{ github.repository_owner }}/awx_devel:${{ env.BRANCH }} make ui-lint - ui-test: - runs-on: ubuntu-latest - permissions: - packages: write - contents: read - steps: - - uses: actions/checkout@v2 - - - name: Log in to registry - run: | - echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin - - - name: Pre-pull image to warm build cache - run: | - docker pull ghcr.io/${{ github.repository_owner }}/awx_devel:${{ env.BRANCH }} || : - - - name: Build image - run: | - DEV_DOCKER_TAG_BASE=ghcr.io/${{ github.repository_owner }} COMPOSE_TAG=${{ env.BRANCH }} make docker-compose-build - - - name: Run UI Tests - run: | - docker run -u $(id -u) --rm -v ${{ github.workspace}}:/awx_devel/:Z \ - --workdir=/awx_devel ghcr.io/${{ github.repository_owner }}/awx_devel:${{ env.BRANCH }} make ui-test awx-operator: runs-on: ubuntu-latest steps: diff --git a/.github/workflows/devel_image.yml b/.github/workflows/devel_image.yml index b13bb4e92e..dc9503c16a 100644 --- a/.github/workflows/devel_image.yml +++ b/.github/workflows/devel_image.yml @@ -13,6 +13,14 @@ jobs: steps: - uses: actions/checkout@v2 + - name: Get python version from Makefile + run: echo py_version=`make PYTHON_VERSION` >> $GITHUB_ENV + + - name: Install python ${{ env.py_version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ env.py_version }} + - name: Log in to registry run: | echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin diff --git a/.github/workflows/e2e_test.yml b/.github/workflows/e2e_test.yml index 8478613b07..bb54044d4f 100644 --- a/.github/workflows/e2e_test.yml +++ b/.github/workflows/e2e_test.yml @@ -18,6 +18,14 @@ jobs: steps: - uses: actions/checkout@v2 + - name: Get python version from Makefile + run: echo py_version=`make PYTHON_VERSION` >> $GITHUB_ENV + + - name: Install python ${{ env.py_version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ env.py_version }} + - name: Install system deps run: sudo apt-get install -y gettext diff --git a/.github/workflows/stage.yml b/.github/workflows/stage.yml index 58baef5b96..64f4e48ac7 100644 --- a/.github/workflows/stage.yml +++ b/.github/workflows/stage.yml @@ -43,6 +43,14 @@ jobs: with: path: awx + - name: Get python version from Makefile + run: echo py_version=`make PYTHON_VERSION` >> $GITHUB_ENV + + - name: Install python ${{ env.py_version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ env.py_version }} + - name: Checkout awx-logos uses: actions/checkout@v2 with: diff --git a/.github/workflows/upload_schema.yml b/.github/workflows/upload_schema.yml index 0c4d266bf6..52d98512e5 100644 --- a/.github/workflows/upload_schema.yml +++ b/.github/workflows/upload_schema.yml @@ -14,6 +14,14 @@ jobs: steps: - uses: actions/checkout@v2 + - name: Get python version from Makefile + run: echo py_version=`make PYTHON_VERSION` >> $GITHUB_ENV + + - name: Install python ${{ env.py_version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ env.py_version }} + - name: Log in to registry run: | echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin diff --git a/Makefile b/Makefile index a92267a632..2b9372b8d1 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,4 @@ PYTHON ?= python3.9 -PYTHON_VERSION = $(shell $(PYTHON) -c "from distutils.sysconfig import get_python_version; print(get_python_version())") OFFICIAL ?= no NODE ?= node NPM_BIN ?= npm @@ -44,7 +43,7 @@ I18N_FLAG_FILE = .i18n_built receiver test test_unit test_coverage coverage_html \ dev_build release_build sdist \ ui-release ui-devel \ - VERSION docker-compose-sources \ + VERSION PYTHON_VERSION docker-compose-sources \ .git/hooks/pre-commit clean-tmp: @@ -266,7 +265,7 @@ api-lint: awx-link: [ -d "/awx_devel/awx.egg-info" ] || $(PYTHON) /awx_devel/setup.py egg_info_dev - cp -f /tmp/awx.egg-link /var/lib/awx/venv/awx/lib/python$(PYTHON_VERSION)/site-packages/awx.egg-link + cp -f /tmp/awx.egg-link /var/lib/awx/venv/awx/lib/$(PYTHON)/site-packages/awx.egg-link TEST_DIRS ?= awx/main/tests/unit awx/main/tests/functional awx/conf/tests awx/sso/tests @@ -530,6 +529,9 @@ psql-container: VERSION: @echo "awx: $(VERSION)" +PYTHON_VERSION: + @echo "$(PYTHON)" | sed 's:python::' + Dockerfile: tools/ansible/roles/dockerfile/templates/Dockerfile.j2 ansible-playbook tools/ansible/dockerfile.yml -e receptor_image=$(RECEPTOR_IMAGE) diff --git a/pre-commit.sh b/pre-commit.sh index 4f073799a5..80a49fb47c 100755 --- a/pre-commit.sh +++ b/pre-commit.sh @@ -1,6 +1,8 @@ -if [ -z $AWX_IGNORE_BLACK ] -then - black --check $(git diff --cached --name-only --diff-filter=AM | grep -E '\.py') || \ - (echo 'To fix this, run `make black` to auto-format your code prior to commit, or set AWX_IGNORE_BLACK=1' && \ - exit 1) +if [ -z $AWX_IGNORE_BLACK ] ; then + python_files_changed=$(git diff --cached --name-only --diff-filter=AM | grep -E '\.py') + if [ "x$python_files_changed" != "x" ] ; then + black --check $python_files_changed || \ + (echo 'To fix this, run `make black` to auto-format your code prior to commit, or set AWX_IGNORE_BLACK=1' && \ + exit 1) + fi fi