Upgrade ansible (#10190)

* project: update all dependencies including ansible

Upgrade to ansible 7.x and ansible-core 2.14.x. There seems to be issue
with ansible 8/ansible-core 2.15 so we remain on those versions for now.
It's quite a big bump already anyway.

Signed-off-by: Arthur Outhenin-Chalandre <arthur.outhenin-chalandre@proton.ch>

* tests: install aws galaxy collection

Signed-off-by: Arthur Outhenin-Chalandre <arthur.outhenin-chalandre@proton.ch>

* ansible-lint: disable various rules after ansible upgrade

Temporarily disable a bunch of linting action following ansible upgrade.
Those should be taken care of separately.

Signed-off-by: Arthur Outhenin-Chalandre <arthur.outhenin-chalandre@proton.ch>

* project: resolve deprecated-module ansible-lint error

Signed-off-by: Arthur Outhenin-Chalandre <arthur.outhenin-chalandre@proton.ch>

* project: resolve no-free-form ansible-lint error

Signed-off-by: Arthur Outhenin-Chalandre <arthur.outhenin-chalandre@proton.ch>

* project: resolve schema[meta] ansible-lint error

Signed-off-by: Arthur Outhenin-Chalandre <arthur.outhenin-chalandre@proton.ch>

* project: resolve schema[playbook] ansible-lint error

Signed-off-by: Arthur Outhenin-Chalandre <arthur.outhenin-chalandre@proton.ch>

* project: resolve schema[tasks] ansible-lint error

Signed-off-by: Arthur Outhenin-Chalandre <arthur.outhenin-chalandre@proton.ch>

* project: resolve risky-file-permissions ansible-lint error

Signed-off-by: Arthur Outhenin-Chalandre <arthur.outhenin-chalandre@proton.ch>

* project: resolve risky-shell-pipe ansible-lint error

Signed-off-by: Arthur Outhenin-Chalandre <arthur.outhenin-chalandre@proton.ch>

* project: remove deprecated warn args

Signed-off-by: Arthur Outhenin-Chalandre <arthur.outhenin-chalandre@proton.ch>

* project: use fqcn for non builtin tasks

Signed-off-by: Arthur Outhenin-Chalandre <arthur.outhenin-chalandre@proton.ch>

* project: resolve syntax-check[missing-file] for contrib playbook

Signed-off-by: Arthur Outhenin-Chalandre <arthur.outhenin-chalandre@proton.ch>

* project: use arithmetic inside jinja to fix ansible 6 upgrade

Signed-off-by: Arthur Outhenin-Chalandre <arthur.outhenin-chalandre@proton.ch>

---------

Signed-off-by: Arthur Outhenin-Chalandre <arthur.outhenin-chalandre@proton.ch>

tests/cloud_playbooks/create-aws.yml

@@ -5,7 +5,7 @@
 
   tasks:
   - name: Provision a set of instances
-    ec2:
+    amazon.aws.ec2_instance:
       key_name: "{{ aws.key_name }}"
       aws_access_key: "{{ aws.access_key }}"
       aws_secret_key: "{{ aws.secret_key }}"

tests/cloud_playbooks/create-do.yml

@@ -52,7 +52,8 @@
         test_name: "{{ test_id |regex_replace('\\.', '-') }}"
 
     - name: show vars
-      debug: msg="{{ cloud_region }}, {{ cloud_image }}"
+      debug:
+        msg: "{{ cloud_region }}, {{ cloud_image }}"
 
     - name: set instance names
       set_fact:
@@ -64,7 +65,7 @@
           {%- endif -%}
 
     - name: Manage DO instances | {{ state }}
-      digital_ocean:
+      community.digitalocean.digital_ocean:
         unique_name: yes
         api_token: "{{ lookup('env','DO_API_TOKEN') }}"
         command: "droplet"

tests/cloud_playbooks/create-gce.yml

@@ -46,7 +46,9 @@
       register: gce
 
     - name: Add instances to host group
-      add_host: hostname={{ item.public_ip }} groupname="waitfor_hosts"
+      add_host:
+        hostname: "{{ item.public_ip }}"
+        groupname: "waitfor_hosts"
       with_items: '{{ gce.instance_data }}'
 
     - name: Template the inventory  # noqa 404 CI inventory templates are not in role_path

tests/cloud_playbooks/delete-aws.yml

@@ -4,10 +4,10 @@
 
   tasks:
   - name: Gather EC2 facts
-    action: ec2_facts
+    amazon.aws.ec2_metadata_facts:
 
   - name: Terminate EC2 instances
-    ec2:
+    amazon.aws.ec2_instance:
       aws_access_key: "{{ aws_access_key }}"
       aws_secret_key: "{{ aws_secret_key }}"
       state: absent

tests/cloud_playbooks/roles/packet-ci/tasks/delete-vms.yml

@@ -25,6 +25,6 @@
   changed_when:
     - delete_namespace.rc == 0
   retries: 12
-  delay: "10"
+  delay: 10
   until:
     - delete_namespace.rc != 0

tests/cloud_playbooks/upload-logs-gcs.yml

@@ -21,7 +21,7 @@
         file_name: "{{ ostype }}-{{ kube_network_plugin }}-{{ commit }}-logs.tar.gz"
 
     - name: Create a bucket
-      gc_storage:
+      community.google.gc_storage:
         bucket: "{{ test_name }}"
         mode: create
         permission: public-read
@@ -46,11 +46,12 @@
       get_url:
         url: https://dl.google.com/dl/cloudsdk/channels/rapid/install_google_cloud_sdk.bash
         dest: "{{ dir }}/gcp-installer.sh"
+        mode: 0644
 
     - name: Get gsutil tool
-      script: "{{ dir }}/gcp-installer.sh"
+      command: "{{ dir }}/gcp-installer.sh"
       environment:
-        CLOUDSDK_CORE_DISABLE_PROMPTS: 1
+        CLOUDSDK_CORE_DISABLE_PROMPTS: "1"
         CLOUDSDK_INSTALL_DIR: "{{ dir }}"
       no_log: True
       failed_when: false
@@ -63,7 +64,7 @@
       no_log: True
 
     - name: Upload collected diagnostic info
-      gc_storage:
+      community.google.gc_storage:
         bucket: "{{ test_name }}"
         mode: put
         permission: public-read

tests/requirements.txt

@@ -1,11 +1,11 @@
 -r ../requirements.txt
-ansible-lint==5.4.0
+ansible-lint==6.16.2
 apache-libcloud==3.7.0
 ara[server]==1.6.1
 dopy==0.3.7
 molecule==5.0.1
 molecule-plugins[vagrant]==23.4.1
 python-vagrant==1.0.0
-pytest-testinfra==7.0.0
-tox==4.5.1
-yamllint==1.31.0
+pytest-testinfra==8.1.0
+tox==4.5.2
+yamllint==1.32.0

tests/requirements.yml

@@ -0,0 +1,4 @@
+---
+collections:
+  - name: amazon.aws
+    version: 6.0.1

tests/scripts/testcases_prepare.sh

@@ -3,6 +3,7 @@ set -euxo pipefail
 
 /usr/bin/python -m pip uninstall -y ansible ansible-base ansible-core
 /usr/bin/python -m pip install -r tests/requirements.txt
+ansible-galaxy install -r tests/requirements.yml
 mkdir -p /.ssh
 mkdir -p cluster-dump
 mkdir -p $HOME/.ssh

tests/testcases/030_check-network.yml

@@ -32,7 +32,7 @@
         fail_msg: kubelet_rotate_server_certificates is {{ kubelet_rotate_server_certificates }} but no csr's found
 
     - name: Get Denied/Pending certificate signing requests
-      shell: "{{ bin_dir }}/kubectl get csr | grep -e Denied -e Pending || true"
+      shell: "set -o pipefail && {{ bin_dir }}/kubectl get csr | grep -e Denied -e Pending || true"
       register: get_csr_denied_pending
       changed_when: false
 
@@ -87,6 +87,7 @@
   - name: Run 2 agnhost pods in test ns
     shell:
       cmd: |
+        set -o pipefail
         cat <<EOF | {{ bin_dir }}/kubectl apply -f -
         apiVersion: v1
         kind: Pod
@@ -107,6 +108,7 @@
               seccompProfile:
                 type: RuntimeDefault
         EOF
+      executable: /bin/bash
     changed_when: false
     loop:
     - agnhost1

tests/testcases/040_check-network-adv.yml

@@ -173,6 +173,7 @@
       # heuristics by using the cmd parameter like this:
       shell:
         cmd: |
+          set -o pipefail
           cat <<EOF | {{ bin_dir }}/kubectl create -f -
           apiVersion: "k8s.cni.cncf.io/v1"
           kind: NetworkAttachmentDefinition
@@ -196,6 +197,7 @@
             }
           }'
           EOF
+        executable: /bin/bash
       when:
         - inventory_hostname == groups['kube_control_plane'][0]
         - kube_network_plugin_multus|default(false)|bool
@@ -207,6 +209,7 @@
       # heuristics by using the cmd parameter like this:
       shell:
         cmd: |
+          set -o pipefail
           cat <<EOF | {{ bin_dir }}/kubectl create -f -
           apiVersion: v1
           kind: Pod
@@ -220,6 +223,7 @@
               command: ["/bin/bash", "-c", "sleep 2000000000000"]
               image: dougbtv/centos-network
           EOF
+        executable: /bin/bash
       when:
         - inventory_hostname == groups['kube_control_plane'][0]
         - kube_network_plugin_multus|default(false)|bool

tests/testcases/100_check-k8s-conformance.yml

@@ -17,6 +17,7 @@
       get_url:
         url: "https://github.com/heptio/sonobuoy/releases/download/v{{ sonobuoy_version }}/sonobuoy_{{ sonobuoy_version }}_linux_{{ sonobuoy_arch }}.tar.gz"
         dest: /tmp/sonobuoy.tar.gz
+        mode: 0644
 
     - name: Extract sonobuoy
       unarchive:

tests/testcases/roles/cluster-dump/tasks/main.yml

@@ -5,7 +5,7 @@
   when: inventory_hostname in groups['kube_control_plane']
 
 - name: Compress directory cluster-dump
-  archive:
+  community.general.archive:
     path: /tmp/cluster-dump
     dest: /tmp/cluster-dump.tgz
     mode: 0644