Merge pull request #12754 from tico88612/test/hydrophone

Test: refactor check-network to hydrophone

tests/cloud_playbooks/roles/packet-ci/defaults/main.yml

@@ -4,7 +4,7 @@
 vm_cpu_cores: 2
 vm_cpu_sockets: 1
 vm_cpu_threads: 2
-vm_memory: 2048
+vm_memory: 4096
 releases_disk_size: 2Gi
 
 # Request/Limit allocation settings

tests/common_vars.yml

@@ -38,3 +38,9 @@ flannel_image_repo: "{{ quay_image_repo }}/kubespray/flannel"
 flannel_init_image_repo: "{{ quay_image_repo }}/kubespray/flannel-cni-plugin"
 
 local_release_dir: "{{ '/tmp/releases' if inventory_hostname != 'localhost' else (lookup('env', 'PWD') + '/downloads') }}"
+
+hydrophone_version: "0.7.0"
+hydrophone_arch: "x86_64"
+hydrophone_checksum: "sha256:15a6c09962f9bd4a1587af068b5edef1072327a77012d8fbb84992c7c87c0475"
+hydrophone_parallel: 1
+hydrophone_path: "{{ bin_dir }}/hydrophone"

tests/files/ubuntu22-all-in-one-docker.yml

@@ -2,7 +2,7 @@
 # Instance settings
 cloud_image: ubuntu-2204
 mode: all-in-one
-vm_memory: 1800
+vm_memory: 3072
 
 # Kubespray settings
 auto_renew_certificates: true

tests/files/ubuntu22-calico-all-in-one-upgrade.yml

@@ -2,7 +2,7 @@
 # Instance settings
 cloud_image: ubuntu-2204
 mode: all-in-one
-vm_memory: 1800
+vm_memory: 3072
 
 # Kubespray settings
 auto_renew_certificates: true

tests/files/ubuntu22-calico-all-in-one.yml

@@ -2,7 +2,7 @@
 # Instance settings
 cloud_image: ubuntu-2204
 mode: all-in-one
-vm_memory: 1800
+vm_memory: 3072
 
 # Kubespray settings
 auto_renew_certificates: true

tests/files/ubuntu24-all-in-one-docker.yml

@@ -2,7 +2,7 @@
 # Instance settings
 cloud_image: ubuntu-2404
 mode: all-in-one
-vm_memory: 1800
+vm_memory: 3072
 
 # Kubespray settings
 auto_renew_certificates: true

tests/files/ubuntu24-calico-all-in-one-hardening.yml

@@ -75,7 +75,10 @@ etcd_deployment_type: kubeadm
 kubelet_authentication_token_webhook: true
 kube_read_only_port: 0
 kubelet_rotate_server_certificates: true
-kubelet_csr_approver_enabled: false
+kubelet_csr_approver_enabled: true # For hydrophone
+kubelet_csr_approver_values:
+  # Do not check DNS resolution in testing (not recommended in production)
+  bypassDnsResolution: true
 kubelet_protect_kernel_defaults: true
 kubelet_event_record_qps: 1
 kubelet_rotate_certificates: true

tests/files/ubuntu24-calico-all-in-one.yml

@@ -2,7 +2,7 @@
 # Instance settings
 cloud_image: ubuntu-2404
 mode: all-in-one
-vm_memory: 1800
+vm_memory: 3072
 
 # Kubespray settings
 auto_renew_certificates: true

tests/files/ubuntu24-calico-etcd-datastore.yml

@@ -2,7 +2,7 @@
 # Instance settings
 cloud_image: ubuntu-2404
 mode: node-etcd-client
-vm_memory: 1800
+vm_memory: 3072
 
 # Kubespray settings
 auto_renew_certificates: true

tests/testcases/000_install-hydrophone.yml

@@ -0,0 +1,13 @@
+---
+- name: Download hydrophone
+  get_url:
+    url: "https://github.com/kubernetes-sigs/hydrophone/releases/download/v{{ hydrophone_version }}/hydrophone_Linux_{{ hydrophone_arch }}.tar.gz"
+    dest: /tmp/hydrophone.tar.gz
+    checksum: "{{ hydrophone_checksum }}"
+    mode: "0644"
+
+- name: Extract hydrophone
+  unarchive:
+    src: /tmp/hydrophone.tar.gz
+    dest: "{{ bin_dir }}"
+    copy: false

tests/testcases/025_check-csr-request.yml

@@ -0,0 +1,48 @@
+---
+- name: Check kubelet serving certificates approved with kubelet_csr_approver
+  when:
+  - kubelet_rotate_server_certificates | default(false)
+  - kubelet_csr_approver_enabled | default(kubelet_rotate_server_certificates | default(false))
+  vars:
+    csrs: "{{ csr_json.stdout | from_json }}"
+  block:
+
+  - name: Get certificate signing requests
+    command: "{{ bin_dir }}/kubectl get csr -o jsonpath-as-json={.items[*]}"
+    register: csr_json
+    changed_when: false
+
+  - name: Check there are csrs
+    assert:
+      that: csrs | length > 0
+      fail_msg: kubelet_rotate_server_certificates is {{ kubelet_rotate_server_certificates }} but no csr's found
+
+  - name: Check there are Denied/Pending csrs
+    assert:
+      that:
+      - csrs | rejectattr('status') | length == 0 # Pending == no status
+      - csrs | map(attribute='status.conditions') | flatten | selectattr('type', 'equalto', 'Denied') | length == 0 # Denied
+
+      fail_msg: kubelet_csr_approver is enabled but CSRs are not approved
+
+- name: Approve kubelet serving certificates
+  when:
+  - kubelet_rotate_server_certificates | default(false)
+  - not (kubelet_csr_approver_enabled | default(kubelet_rotate_server_certificates | default(false)))
+  block:
+
+  - name: Get certificate signing requests
+    command: "{{ bin_dir }}/kubectl get csr -o name"
+    register: get_csr
+    changed_when: false
+
+  - name: Check there are csrs
+    assert:
+      that: get_csr.stdout_lines | length > 0
+      fail_msg: kubelet_rotate_server_certificates is {{ kubelet_rotate_server_certificates }} but no csr's found
+
+  - name: Approve certificates
+    command: "{{ bin_dir }}/kubectl certificate approve {{ get_csr.stdout_lines | join(' ') }}"
+    register: certificate_approve
+    when: get_csr.stdout_lines | length > 0
+    changed_when: certificate_approve.stdout

tests/testcases/030_check-network.yml

@@ -1,114 +1,10 @@
 ---
-- name: Check kubelet serving certificates approved with kubelet_csr_approver
+- name: Run the hydrophone checks
-  when:
-  - kubelet_rotate_server_certificates | default(false)
-  - kubelet_csr_approver_enabled | default(kubelet_rotate_server_certificates | default(false))
   vars:
-    csrs: "{{ csr_json.stdout | from_json }}"
+    networking_check: "\\[sig-network\\] Networking Granular Checks.+\\[Conformance\\]"
   block:
-
+  - name: Run the networking granular checks
-  - name: Get certificate signing requests
+    command: "{{ hydrophone_path }} --focus=\"{{ networking_check }}\" --parallel {{ hydrophone_parallel }}"
-    command: "{{ bin_dir }}/kubectl get csr -o jsonpath-as-json={.items[*]}"
-    register: csr_json
-    changed_when: false
-
-  - name: Check there are csrs
-    assert:
-      that: csrs | length > 0
-      fail_msg: kubelet_rotate_server_certificates is {{ kubelet_rotate_server_certificates }} but no csr's found
-
-  - name: Check there are Denied/Pending csrs
-    assert:
-      that:
-      - csrs | rejectattr('status') | length == 0 # Pending == no status
-      - csrs | map(attribute='status.conditions') | flatten | selectattr('type', 'equalto', 'Denied') | length == 0 # Denied
-
-      fail_msg: kubelet_csr_approver is enabled but CSRs are not approved
-
-- name: Approve kubelet serving certificates
-  when:
-  - kubelet_rotate_server_certificates | default(false)
-  - not (kubelet_csr_approver_enabled | default(kubelet_rotate_server_certificates | default(false)))
-  block:
-
-  - name: Get certificate signing requests
-    command: "{{ bin_dir }}/kubectl get csr -o name"
-    register: get_csr
-    changed_when: false
-
-  - name: Check there are csrs
-    assert:
-      that: get_csr.stdout_lines | length > 0
-      fail_msg: kubelet_rotate_server_certificates is {{ kubelet_rotate_server_certificates }} but no csr's found
-
-  - name: Approve certificates
-    command: "{{ bin_dir }}/kubectl certificate approve {{ get_csr.stdout_lines | join(' ') }}"
-    register: certificate_approve
-    when: get_csr.stdout_lines | length > 0
-    changed_when: certificate_approve.stdout
-
-- name: Create test namespace
-  command: "{{ bin_dir }}/kubectl create namespace test"
-  changed_when: false
-
-- name: Run 2 agnhost pods in test ns
-  command:
-    cmd: "{{ bin_dir }}/kubectl apply --namespace test -f -"
-    stdin: |
-      apiVersion: apps/v1
-      kind: Deployment
-      metadata:
-        name: agnhost
-      spec:
-        replicas: 2
-        selector:
-          matchLabels:
-            app: agnhost
-        template:
-          metadata:
-            labels:
-              app: agnhost
-          spec:
-            containers:
-            - name: agnhost
-              image: {{ test_image_repo }}:{{ test_image_tag }}
-              command: ['/agnhost', 'netexec', '--http-port=8080']
-              securityContext:
-                allowPrivilegeEscalation: false
-                capabilities:
-                  drop: ['ALL']
-                runAsUser: 1000
-                runAsNonRoot: true
-                seccompProfile:
-                  type: RuntimeDefault
-  changed_when: false
-
-- name: Check that all pods are running and ready
-  vars:
-    pods: "{{ (pods_json.stdout | from_json)['items'] }}"
-  block:
-  - name: Check Deployment is ready
-    command: "{{ bin_dir }}/kubectl rollout status deploy --namespace test agnhost --timeout=180s"
-    changed_when: false
-  - name: Get pod names
-    command: "{{ bin_dir }}/kubectl get pods -n test -o json"
-    changed_when: false
-    register: pods_json
-
-  - name: Check pods IP are in correct network
-    assert:
-      that: pods
-            | selectattr('status.phase', '==', 'Running')
-            | selectattr('status.podIP', 'ansible.utils.in_network', kube_pods_subnet)
-            | length == 2
-
-  - name: Curl between pods is working
-    command: "{{ bin_dir }}/kubectl -n test exec {{ item[0].metadata.name }} -- curl {{ item[1].status.podIP | ansible.utils.ipwrap}}:8080"
-    with_nested:
-    - "{{ pods }}"
-    - "{{ pods }}"
-    loop_control:
-      label: "{{ item[0].metadata.name + ' --> ' + item[1].metadata.name }}"
   rescue:
   - name: List pods cluster-wide
     command: "{{ bin_dir }}/kubectl get pods --all-namespaces -owide"

tests/testcases/tests.yml

@@ -11,6 +11,8 @@
     - name: Import Kubespray variables
       import_role:
         name: ../../roles/kubespray_defaults
+    - name: Install the Hydrophone for tests
+      import_tasks: 000_install-hydrophone.yml
     - name: Testcases for apiserver
       import_tasks: 010_check-apiserver.yml
       when:
@@ -24,12 +26,11 @@
         - name: Testcases checking pods
           import_tasks: 020_check-pods-running.yml
           when: ('macvlan' not in testcase)
+        - name: Checking CSR approver
+          import_tasks: 025_check-csr-request.yml
         - name: Testcases for network
           import_tasks: 030_check-network.yml
           when: ('macvlan' not in testcase)
-          vars:
-            test_image_repo: registry.k8s.io/e2e-test-images/agnhost
-            test_image_tag: "2.40"
     - name: Testcases for calico / advanced network
       import_tasks: 040_check-network-adv.yml
       when: