ci: run the Ansible controller as an unprivileged user (#13527)

* ci: run the Ansible controller as an unprivileged user

* ci: clarify non-root controller check

* ci: make the sudo privilege check fail when sudo succeeds

Bash does not apply errexit to the negated sudo command. The old check could not fail the job even when passwordless sudo worked. Exit explicitly when sudo succeeds.

Remove --break-system-packages from both upgrade pip installs because they run inside the controller virtual environment.

.gitlab-ci.yml

@@ -8,11 +8,19 @@ stages:
 variables:
   FAILFASTCI_NAMESPACE: 'kargo-ci'
   GITLAB_REPOSITORY: 'kargo-ci/kubernetes-sigs-kubespray'
-  GIT_CONFIG_COUNT: 2
+  # Let the helper assign checkout ownership to the build user.
+  FF_DISABLE_UMASK_FOR_DOCKER_EXECUTOR: "true"
+  FF_DISABLE_UMASK_FOR_KUBERNETES_EXECUTOR: "true"
+  GIT_CONFIG_COUNT: 3
   GIT_CONFIG_KEY_0: user.email
   GIT_CONFIG_VALUE_0: "ci@kubespray.io"
   GIT_CONFIG_KEY_1: user.name
   GIT_CONFIG_VALUE_1: "Kubespray CI"
+  # TODO: Remove this safe.directory workaround after fixing the inconsistent GitLab Runner versions.
+  # Runners older than 17.11 may leave checkout ownership incorrect despite the FF_DISABLE_UMASK flags.
+  # https://github.com/kubernetes-sigs/kubespray/pull/13527#discussion_r4198076964
+  GIT_CONFIG_KEY_2: safe.directory
+  GIT_CONFIG_VALUE_2: "$CI_PROJECT_DIR"
   ANSIBLE_FORCE_COLOR: "true"
   MAGIC: "ci check this"
   GS_ACCESS_KEY_ID: $GS_KEY

.gitlab-ci/lint.yml

@@ -16,11 +16,24 @@ pre-commit:
     when: 'always'
   needs: []
 
+# Verify non-root Git setup, CI file writes, and local Ansible execution.
+non-root-controller:
+  extends: .job
+  stage: test
+  script:
+  - test "$(id -u)" -ne 0
+  - if sudo -n true; then exit 1; fi
+  - test "$HOME" = /home/kubespray
+  - |
+    for directory in "$HOME" "$CI_PROJECT_DIR" "$CI_PROJECT_DIR/.git" "$CI_PROJECT_DIR"/contrib/terraform/*/; do
+      probe=$(mktemp "$directory/.write-check.XXXXXX")
+      rm "$probe"
+    done
+  - ansible localhost --connection=local -m setup -a 'filter=ansible_user_uid'
+
 vagrant-validate:
   extends: .job
   stage: test
   tags: [ffci]
-  variables:
-    VAGRANT_VERSION: 2.3.7
   script:
   - ./tests/scripts/vagrant-validate.sh

pipeline.Dockerfile

@@ -16,6 +16,7 @@ RUN apt update -q \
          libssl-dev \
          python3-dev \
          python3-pip \
+         python3-venv \
          sshpass \
          apt-transport-https \
          jq \
@@ -44,16 +45,22 @@ ADD ./requirements.txt  /kubespray/requirements.txt
 ADD ./tests/requirements.txt /kubespray/tests/requirements.txt
 
 RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 1 \
-    && pip install --break-system-packages --ignore-installed --no-compile --no-cache-dir pip -U \
-    && pip install --break-system-packages --no-compile --no-cache-dir -r tests/requirements.txt \
     && curl -L https://dl.k8s.io/release/v1.37.1/bin/linux/$(dpkg --print-architecture)/kubectl -o /usr/local/bin/kubectl \
     && echo $(curl -L https://dl.k8s.io/release/v1.37.1/bin/linux/$(dpkg --print-architecture)/kubectl.sha256) /usr/local/bin/kubectl | sha256sum --check \
     && chmod a+x /usr/local/bin/kubectl \
     # Install Vagrant
     && curl -LO https://releases.hashicorp.com/vagrant/${VAGRANT_VERSION}/vagrant_${VAGRANT_VERSION}-1_$(dpkg --print-architecture).deb \
     && dpkg -i vagrant_${VAGRANT_VERSION}-1_$(dpkg --print-architecture).deb \
-    && rm vagrant_${VAGRANT_VERSION}-1_$(dpkg --print-architecture).deb \
+    && rm vagrant_${VAGRANT_VERSION}-1_$(dpkg --print-architecture).deb
+
+RUN useradd --create-home --uid 1001 kubespray
+ENV HOME=/home/kubespray \
+    PATH="/home/kubespray/.local/bin:/home/kubespray/venv/bin:${PATH}"
+USER kubespray
+
+RUN python3 -m venv /home/kubespray/venv \
+    && pip install --no-compile --no-cache-dir -r tests/requirements.txt \
     && vagrant plugin install vagrant-libvirt \
     # Install Kubernetes collections
-    && pip install --break-system-packages --no-compile --no-cache-dir kubernetes==35.0.0 \
+    && pip install --no-compile --no-cache-dir kubernetes==35.0.0 \
     && ansible-galaxy collection install kubernetes.core:==6.4.0

scripts/pipeline.Dockerfile.j2

@@ -16,6 +16,7 @@ RUN apt update -q \
          libssl-dev \
          python3-dev \
          python3-pip \
+         python3-venv \
          sshpass \
          apt-transport-https \
          jq \
@@ -44,16 +45,22 @@ ADD ./requirements.txt  /kubespray/requirements.txt
 ADD ./tests/requirements.txt /kubespray/tests/requirements.txt
 
 RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 1 \
-    && pip install --break-system-packages --ignore-installed --no-compile --no-cache-dir pip -U \
-    && pip install --break-system-packages --no-compile --no-cache-dir -r tests/requirements.txt \
     && curl -L https://dl.k8s.io/release/v{{ kube_version }}/bin/linux/$(dpkg --print-architecture)/kubectl -o /usr/local/bin/kubectl \
     && echo $(curl -L https://dl.k8s.io/release/v{{ kube_version }}/bin/linux/$(dpkg --print-architecture)/kubectl.sha256) /usr/local/bin/kubectl | sha256sum --check \
     && chmod a+x /usr/local/bin/kubectl \
     # Install Vagrant
     && curl -LO https://releases.hashicorp.com/vagrant/${VAGRANT_VERSION}/vagrant_${VAGRANT_VERSION}-1_$(dpkg --print-architecture).deb \
     && dpkg -i vagrant_${VAGRANT_VERSION}-1_$(dpkg --print-architecture).deb \
-    && rm vagrant_${VAGRANT_VERSION}-1_$(dpkg --print-architecture).deb \
+    && rm vagrant_${VAGRANT_VERSION}-1_$(dpkg --print-architecture).deb
+
+RUN useradd --create-home --uid 1001 kubespray
+ENV HOME=/home/kubespray \
+    PATH="/home/kubespray/.local/bin:/home/kubespray/venv/bin:${PATH}"
+USER kubespray
+
+RUN python3 -m venv /home/kubespray/venv \
+    && pip install --no-compile --no-cache-dir -r tests/requirements.txt \
     && vagrant plugin install vagrant-libvirt \
     # Install Kubernetes collections
-    && pip install --break-system-packages --no-compile --no-cache-dir kubernetes==35.0.0 \
+    && pip install --no-compile --no-cache-dir kubernetes==35.0.0 \
     && ansible-galaxy collection install kubernetes.core:==6.4.0

tests/cloud_playbooks/create-kubevirt.yml

@@ -2,7 +2,7 @@
 - name: Provision Packet VMs
   hosts: localhost
   gather_facts: false
-  become: true
+  become: false
   tasks:
   - name: Create Kubevirt VMs
     import_role:

tests/scripts/opentofu_install.sh

@@ -3,6 +3,6 @@ set -euxo pipefail
 
 curl --proto '=https' --tlsv1.2 -fsSL https://get.opentofu.org/install-opentofu.sh -o install-opentofu.sh
 chmod +x install-opentofu.sh
-./install-opentofu.sh --install-method standalone
+./install-opentofu.sh --install-method standalone --install-path "${HOME}/.local/bin" --symlink-path - --root-method none
 rm -f install-opentofu.sh
 tofu --version

tests/scripts/testcases_run.sh

@@ -18,7 +18,7 @@ if [ "${UPGRADE_TEST}" != "false" ]; then
   # Checkout the current tests/ directory ; even when testing old version,
   # we want the up-to-date test setup/provisionning
   git checkout "${CI_COMMIT_SHA}" -- tests/
-  pip install --break-system-packages --no-compile --no-cache-dir -r requirements.txt
+  pip install --no-compile --no-cache-dir -r requirements.txt
 fi
 
 export ANSIBLE_BECOME=true
@@ -58,7 +58,7 @@ fi
 if [ "${UPGRADE_TEST}" != "false" ]; then
   git checkout "${CI_COMMIT_SHA}"
 
-  pip install --break-system-packages --no-compile --no-cache-dir -r requirements.txt
+  pip install --no-compile --no-cache-dir -r requirements.txt
 
   case "${UPGRADE_TEST}" in
     "basic")

tests/scripts/vagrant-validate.sh

@@ -1,6 +1,4 @@
 #!/bin/bash
 set -euxo pipefail
 
-curl -sL "https://releases.hashicorp.com/vagrant/${VAGRANT_VERSION}/vagrant_${VAGRANT_VERSION}-1_amd64.deb" -o "/tmp/vagrant_${VAGRANT_VERSION}-1_amd64.deb"
-dpkg -i "/tmp/vagrant_${VAGRANT_VERSION}-1_amd64.deb"
 vagrant validate --ignore-provider