diff --git a/tests/cloud_playbooks/roles/packet-ci/defaults/main.yml b/tests/cloud_playbooks/roles/packet-ci/defaults/main.yml index 2c3377fd7..7426ede94 100644 --- a/tests/cloud_playbooks/roles/packet-ci/defaults/main.yml +++ b/tests/cloud_playbooks/roles/packet-ci/defaults/main.yml @@ -4,7 +4,7 @@ vm_cpu_cores: 2 vm_cpu_sockets: 1 vm_cpu_threads: 2 -vm_memory: 2048 +vm_memory: 4096 releases_disk_size: 2Gi # Request/Limit allocation settings diff --git a/tests/common_vars.yml b/tests/common_vars.yml index 4575c607b..822ba654d 100644 --- a/tests/common_vars.yml +++ b/tests/common_vars.yml @@ -38,3 +38,9 @@ flannel_image_repo: "{{ quay_image_repo }}/kubespray/flannel" flannel_init_image_repo: "{{ quay_image_repo }}/kubespray/flannel-cni-plugin" local_release_dir: "{{ '/tmp/releases' if inventory_hostname != 'localhost' else (lookup('env', 'PWD') + '/downloads') }}" + +hydrophone_version: "0.7.0" +hydrophone_arch: "x86_64" +hydrophone_checksum: "sha256:15a6c09962f9bd4a1587af068b5edef1072327a77012d8fbb84992c7c87c0475" +hydrophone_parallel: 1 +hydrophone_path: "{{ bin_dir }}/hydrophone" diff --git a/tests/files/ubuntu22-all-in-one-docker.yml b/tests/files/ubuntu22-all-in-one-docker.yml index 48d3a9495..0120feca7 100644 --- a/tests/files/ubuntu22-all-in-one-docker.yml +++ b/tests/files/ubuntu22-all-in-one-docker.yml @@ -2,7 +2,7 @@ # Instance settings cloud_image: ubuntu-2204 mode: all-in-one -vm_memory: 1800 +vm_memory: 3072 # Kubespray settings auto_renew_certificates: true diff --git a/tests/files/ubuntu22-calico-all-in-one-upgrade.yml b/tests/files/ubuntu22-calico-all-in-one-upgrade.yml index 0f8a83d25..a19321fca 100644 --- a/tests/files/ubuntu22-calico-all-in-one-upgrade.yml +++ b/tests/files/ubuntu22-calico-all-in-one-upgrade.yml @@ -2,7 +2,7 @@ # Instance settings cloud_image: ubuntu-2204 mode: all-in-one -vm_memory: 1800 +vm_memory: 3072 # Kubespray settings auto_renew_certificates: true diff --git a/tests/files/ubuntu22-calico-all-in-one.yml b/tests/files/ubuntu22-calico-all-in-one.yml index ff157b241..3d59c96a0 100644 --- a/tests/files/ubuntu22-calico-all-in-one.yml +++ b/tests/files/ubuntu22-calico-all-in-one.yml @@ -2,7 +2,7 @@ # Instance settings cloud_image: ubuntu-2204 mode: all-in-one -vm_memory: 1800 +vm_memory: 3072 # Kubespray settings auto_renew_certificates: true diff --git a/tests/files/ubuntu24-all-in-one-docker.yml b/tests/files/ubuntu24-all-in-one-docker.yml index f8979fc19..6fe04fac2 100644 --- a/tests/files/ubuntu24-all-in-one-docker.yml +++ b/tests/files/ubuntu24-all-in-one-docker.yml @@ -2,7 +2,7 @@ # Instance settings cloud_image: ubuntu-2404 mode: all-in-one -vm_memory: 1800 +vm_memory: 3072 # Kubespray settings auto_renew_certificates: true diff --git a/tests/files/ubuntu24-calico-all-in-one-hardening.yml b/tests/files/ubuntu24-calico-all-in-one-hardening.yml index 024f320d3..7a6a63f50 100644 --- a/tests/files/ubuntu24-calico-all-in-one-hardening.yml +++ b/tests/files/ubuntu24-calico-all-in-one-hardening.yml @@ -75,7 +75,10 @@ etcd_deployment_type: kubeadm kubelet_authentication_token_webhook: true kube_read_only_port: 0 kubelet_rotate_server_certificates: true -kubelet_csr_approver_enabled: false +kubelet_csr_approver_enabled: true # For hydrophone +kubelet_csr_approver_values: + # Do not check DNS resolution in testing (not recommended in production) + bypassDnsResolution: true kubelet_protect_kernel_defaults: true kubelet_event_record_qps: 1 kubelet_rotate_certificates: true diff --git a/tests/files/ubuntu24-calico-all-in-one.yml b/tests/files/ubuntu24-calico-all-in-one.yml index 4203172bd..664d6151f 100644 --- a/tests/files/ubuntu24-calico-all-in-one.yml +++ b/tests/files/ubuntu24-calico-all-in-one.yml @@ -2,7 +2,7 @@ # Instance settings cloud_image: ubuntu-2404 mode: all-in-one -vm_memory: 1800 +vm_memory: 3072 # Kubespray settings auto_renew_certificates: true diff --git a/tests/files/ubuntu24-calico-etcd-datastore.yml b/tests/files/ubuntu24-calico-etcd-datastore.yml index 738ec8f9a..2b84e3486 100644 --- a/tests/files/ubuntu24-calico-etcd-datastore.yml +++ b/tests/files/ubuntu24-calico-etcd-datastore.yml @@ -2,7 +2,7 @@ # Instance settings cloud_image: ubuntu-2404 mode: node-etcd-client -vm_memory: 1800 +vm_memory: 3072 # Kubespray settings auto_renew_certificates: true diff --git a/tests/testcases/000_install-hydrophone.yml b/tests/testcases/000_install-hydrophone.yml new file mode 100644 index 000000000..b2a476bb4 --- /dev/null +++ b/tests/testcases/000_install-hydrophone.yml @@ -0,0 +1,13 @@ +--- +- name: Download hydrophone + get_url: + url: "https://github.com/kubernetes-sigs/hydrophone/releases/download/v{{ hydrophone_version }}/hydrophone_Linux_{{ hydrophone_arch }}.tar.gz" + dest: /tmp/hydrophone.tar.gz + checksum: "{{ hydrophone_checksum }}" + mode: "0644" + +- name: Extract hydrophone + unarchive: + src: /tmp/hydrophone.tar.gz + dest: "{{ bin_dir }}" + copy: false diff --git a/tests/testcases/025_check-csr-request.yml b/tests/testcases/025_check-csr-request.yml new file mode 100644 index 000000000..f5cae20a0 --- /dev/null +++ b/tests/testcases/025_check-csr-request.yml @@ -0,0 +1,48 @@ +--- +- name: Check kubelet serving certificates approved with kubelet_csr_approver + when: + - kubelet_rotate_server_certificates | default(false) + - kubelet_csr_approver_enabled | default(kubelet_rotate_server_certificates | default(false)) + vars: + csrs: "{{ csr_json.stdout | from_json }}" + block: + + - name: Get certificate signing requests + command: "{{ bin_dir }}/kubectl get csr -o jsonpath-as-json={.items[*]}" + register: csr_json + changed_when: false + + - name: Check there are csrs + assert: + that: csrs | length > 0 + fail_msg: kubelet_rotate_server_certificates is {{ kubelet_rotate_server_certificates }} but no csr's found + + - name: Check there are Denied/Pending csrs + assert: + that: + - csrs | rejectattr('status') | length == 0 # Pending == no status + - csrs | map(attribute='status.conditions') | flatten | selectattr('type', 'equalto', 'Denied') | length == 0 # Denied + + fail_msg: kubelet_csr_approver is enabled but CSRs are not approved + +- name: Approve kubelet serving certificates + when: + - kubelet_rotate_server_certificates | default(false) + - not (kubelet_csr_approver_enabled | default(kubelet_rotate_server_certificates | default(false))) + block: + + - name: Get certificate signing requests + command: "{{ bin_dir }}/kubectl get csr -o name" + register: get_csr + changed_when: false + + - name: Check there are csrs + assert: + that: get_csr.stdout_lines | length > 0 + fail_msg: kubelet_rotate_server_certificates is {{ kubelet_rotate_server_certificates }} but no csr's found + + - name: Approve certificates + command: "{{ bin_dir }}/kubectl certificate approve {{ get_csr.stdout_lines | join(' ') }}" + register: certificate_approve + when: get_csr.stdout_lines | length > 0 + changed_when: certificate_approve.stdout diff --git a/tests/testcases/030_check-network.yml b/tests/testcases/030_check-network.yml index 28d869efc..2acae6e8e 100644 --- a/tests/testcases/030_check-network.yml +++ b/tests/testcases/030_check-network.yml @@ -1,114 +1,10 @@ --- -- name: Check kubelet serving certificates approved with kubelet_csr_approver - when: - - kubelet_rotate_server_certificates | default(false) - - kubelet_csr_approver_enabled | default(kubelet_rotate_server_certificates | default(false)) +- name: Run the hydrophone checks vars: - csrs: "{{ csr_json.stdout | from_json }}" + networking_check: "\\[sig-network\\] Networking Granular Checks.+\\[Conformance\\]" block: - - - name: Get certificate signing requests - command: "{{ bin_dir }}/kubectl get csr -o jsonpath-as-json={.items[*]}" - register: csr_json - changed_when: false - - - name: Check there are csrs - assert: - that: csrs | length > 0 - fail_msg: kubelet_rotate_server_certificates is {{ kubelet_rotate_server_certificates }} but no csr's found - - - name: Check there are Denied/Pending csrs - assert: - that: - - csrs | rejectattr('status') | length == 0 # Pending == no status - - csrs | map(attribute='status.conditions') | flatten | selectattr('type', 'equalto', 'Denied') | length == 0 # Denied - - fail_msg: kubelet_csr_approver is enabled but CSRs are not approved - -- name: Approve kubelet serving certificates - when: - - kubelet_rotate_server_certificates | default(false) - - not (kubelet_csr_approver_enabled | default(kubelet_rotate_server_certificates | default(false))) - block: - - - name: Get certificate signing requests - command: "{{ bin_dir }}/kubectl get csr -o name" - register: get_csr - changed_when: false - - - name: Check there are csrs - assert: - that: get_csr.stdout_lines | length > 0 - fail_msg: kubelet_rotate_server_certificates is {{ kubelet_rotate_server_certificates }} but no csr's found - - - name: Approve certificates - command: "{{ bin_dir }}/kubectl certificate approve {{ get_csr.stdout_lines | join(' ') }}" - register: certificate_approve - when: get_csr.stdout_lines | length > 0 - changed_when: certificate_approve.stdout - -- name: Create test namespace - command: "{{ bin_dir }}/kubectl create namespace test" - changed_when: false - -- name: Run 2 agnhost pods in test ns - command: - cmd: "{{ bin_dir }}/kubectl apply --namespace test -f -" - stdin: | - apiVersion: apps/v1 - kind: Deployment - metadata: - name: agnhost - spec: - replicas: 2 - selector: - matchLabels: - app: agnhost - template: - metadata: - labels: - app: agnhost - spec: - containers: - - name: agnhost - image: {{ test_image_repo }}:{{ test_image_tag }} - command: ['/agnhost', 'netexec', '--http-port=8080'] - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: ['ALL'] - runAsUser: 1000 - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - changed_when: false - -- name: Check that all pods are running and ready - vars: - pods: "{{ (pods_json.stdout | from_json)['items'] }}" - block: - - name: Check Deployment is ready - command: "{{ bin_dir }}/kubectl rollout status deploy --namespace test agnhost --timeout=180s" - changed_when: false - - name: Get pod names - command: "{{ bin_dir }}/kubectl get pods -n test -o json" - changed_when: false - register: pods_json - - - name: Check pods IP are in correct network - assert: - that: pods - | selectattr('status.phase', '==', 'Running') - | selectattr('status.podIP', 'ansible.utils.in_network', kube_pods_subnet) - | length == 2 - - - name: Curl between pods is working - command: "{{ bin_dir }}/kubectl -n test exec {{ item[0].metadata.name }} -- curl {{ item[1].status.podIP | ansible.utils.ipwrap}}:8080" - with_nested: - - "{{ pods }}" - - "{{ pods }}" - loop_control: - label: "{{ item[0].metadata.name + ' --> ' + item[1].metadata.name }}" + - name: Run the networking granular checks + command: "{{ hydrophone_path }} --focus=\"{{ networking_check }}\" --parallel {{ hydrophone_parallel }}" rescue: - name: List pods cluster-wide command: "{{ bin_dir }}/kubectl get pods --all-namespaces -owide" diff --git a/tests/testcases/tests.yml b/tests/testcases/tests.yml index 236de0033..59e6e3827 100644 --- a/tests/testcases/tests.yml +++ b/tests/testcases/tests.yml @@ -11,6 +11,8 @@ - name: Import Kubespray variables import_role: name: ../../roles/kubespray_defaults + - name: Install the Hydrophone for tests + import_tasks: 000_install-hydrophone.yml - name: Testcases for apiserver import_tasks: 010_check-apiserver.yml when: @@ -24,12 +26,11 @@ - name: Testcases checking pods import_tasks: 020_check-pods-running.yml when: ('macvlan' not in testcase) + - name: Checking CSR approver + import_tasks: 025_check-csr-request.yml - name: Testcases for network import_tasks: 030_check-network.yml when: ('macvlan' not in testcase) - vars: - test_image_repo: registry.k8s.io/e2e-test-images/agnhost - test_image_tag: "2.40" - name: Testcases for calico / advanced network import_tasks: 040_check-network-adv.yml when: