diff --git a/roles/kubernetes/control-plane/files/k8s-certs-renew.py b/roles/kubernetes/control-plane/files/k8s-certs-renew.py new file mode 100644 index 000000000..89f487255 --- /dev/null +++ b/roles/kubernetes/control-plane/files/k8s-certs-renew.py @@ -0,0 +1,142 @@ +#!/usr/bin/env python3 +"""Renew the certificates managed by kubeadm if they expire before the next +scheduled run of the k8s-certs-renew timer.""" + +import argparse +import json +import os +import shutil +import socket +import subprocess +import sys +import time +from datetime import datetime, timedelta, timezone + +DAYS_BUFFER = 7 # time margin, because we should not renew at the last moment + + +def parse_args(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--kubeadm", required=True, + help="path to the kubeadm binary") + parser.add_argument("--calendar", required=True, + help="systemd calendar spec of the renewal timer") + parser.add_argument("--admin-conf", required=True, + help="path to the kubeadm admin.conf") + parser.add_argument("--apiserver-port", type=int, required=True, + help="local apiserver port to wait on after the restart") + parser.add_argument("--crictl", required=True, + help="path to crictl, used to restart the control plane pods") + return parser.parse_args() + + +def log(message): + print(message, flush=True) + + +def run(cmd, **kwargs): + return subprocess.run(cmd, check=True, text=True, **kwargs) + + +def next_scheduled_run(calendar): + """Next elapse of the renewal timer, or None if it cannot be determined.""" + # systemctl show reports an empty NextElapseUSecRealtime while the timer's + # unit is running, so use the calendar spec to get the next scheduled run. + # TZ=UTC makes the output format the same in all timezones. The offset + # that this causes is small compared to DAYS_BUFFER. + env = dict(os.environ, LC_ALL="C", TZ="UTC") + try: + output = run( + ["systemd-analyze", "calendar", calendar], + stdout=subprocess.PIPE, + env=env, + ).stdout + except (OSError, subprocess.CalledProcessError): + return None + for line in output.splitlines(): + label, _, value = line.partition(":") + if label.strip() == "Next elapse": + try: + # "Mon 2026-10-05 03:00:00 UTC" or "never" + elapse = datetime.strptime(value.strip(), "%a %Y-%m-%d %H:%M:%S UTC") + except ValueError: + return None + return elapse.replace(tzinfo=timezone.utc) + return None + + +def certs_to_renew(kubeadm, threshold): + output = run( + [kubeadm, "certs", "check-expiration", "-o", "json"], + stdout=subprocess.PIPE, + ).stdout + expiring = [] + for cert in json.loads(output)["certificates"]: + # Ignore missing certs, kubeadm cannot renew them. For example, + # super-admin.conf is only on the node where "kubeadm init" ran. + if cert["missing"]: + continue + expiration = datetime.strptime(cert["expirationDate"], "%Y-%m-%dT%H:%M:%S%z") + if expiration < threshold: + expiring.append(cert["name"]) + return expiring + + +def restart_control_plane(crictl): + pods = run( + [crictl, "pods", "--namespace", "kube-system", "-q", + "--name", "kube-scheduler-*|kube-controller-manager-*|kube-apiserver-*|etcd-*"], + stdout=subprocess.PIPE, + ).stdout.split() + if pods: + run([crictl, "rmp", "-f"] + pods) + + +def wait_for_apiserver(port): + while True: + try: + with socket.create_connection(("127.0.0.1", port), timeout=1): + return + except OSError: + time.sleep(1) + + +def main(): + args = parse_args() + + log("Check expiration before renewal") + run([args.kubeadm, "certs", "check-expiration"]) + + next_run = next_scheduled_run(args.calendar) + if next_run is None: + log("Cannot get the next scheduled run from the systemd calendar, renew directly") + else: + expiring = certs_to_renew(args.kubeadm, next_run + timedelta(days=DAYS_BUFFER)) + if not expiring: + log("Skip cert renew and K8S container restart, since all certificates expire after the next scheduled run") + return + log("Certificates to renew:") + log("\n".join(expiring)) + + log("Renewing certificates managed by kubeadm") + run([args.kubeadm, "certs", "renew", "all"]) + + log("Restarting control plane pods managed by kubeadm") + restart_control_plane(args.crictl) + + log("Updating /root/.kube/config") + shutil.copy(args.admin_conf, "/root/.kube/config") + + log("Waiting for apiserver to be up again") + wait_for_apiserver(args.apiserver_port) + + log("Expiration after renewal") + run([args.kubeadm, "certs", "check-expiration"]) + + +if __name__ == "__main__": + try: + main() + except subprocess.CalledProcessError as err: + log(f"Command {err.cmd} failed with return code {err.returncode}") + sys.exit(err.returncode) diff --git a/roles/kubernetes/control-plane/tasks/main.yml b/roles/kubernetes/control-plane/tasks/main.yml index 3f7933c7a..49c7f7868 100644 --- a/roles/kubernetes/control-plane/tasks/main.yml +++ b/roles/kubernetes/control-plane/tasks/main.yml @@ -127,9 +127,9 @@ when: kube_apiserver_use_authorization_config_file - name: Install script to renew K8S control plane certificates - template: - src: k8s-certs-renew.sh.j2 - dest: "{{ bin_dir }}/k8s-certs-renew.sh" + copy: + src: k8s-certs-renew.py + dest: "{{ bin_dir }}/k8s-certs-renew.py" mode: "0755" - name: Renew K8S control plane certificates monthly 1/2 @@ -153,3 +153,10 @@ state: started daemon_reload: "{{ k8s_certs_units is changed }}" when: auto_renew_certificates + +# TODO: remove after the 2.33.0 release +- name: Remove the previous shell version of the renewal script + file: + path: "{{ bin_dir }}/k8s-certs-renew.sh" + state: absent + when: auto_renew_certificates diff --git a/roles/kubernetes/control-plane/templates/k8s-certs-renew.service.j2 b/roles/kubernetes/control-plane/templates/k8s-certs-renew.service.j2 index 64610c2bc..f2bf97ecf 100644 --- a/roles/kubernetes/control-plane/templates/k8s-certs-renew.service.j2 +++ b/roles/kubernetes/control-plane/templates/k8s-certs-renew.service.j2 @@ -3,4 +3,9 @@ Description=Renew K8S control plane certificates [Service] Type=oneshot -ExecStart={{ bin_dir }}/k8s-certs-renew.sh +ExecStart={{ '/opt/bin/python' if ansible_facts['os_family'] == 'Flatcar' else '/usr/bin/python3' }} {{ bin_dir }}/k8s-certs-renew.py \ + --kubeadm {{ bin_dir }}/kubeadm \ + --calendar "{{ auto_renew_certificates_systemd_calendar }}" \ + --admin-conf {{ kube_config_dir }}/admin.conf \ + --apiserver-port {{ kube_apiserver_port }} \ + --crictl {{ bin_dir }}/crictl diff --git a/roles/kubernetes/control-plane/templates/k8s-certs-renew.sh.j2 b/roles/kubernetes/control-plane/templates/k8s-certs-renew.sh.j2 deleted file mode 100644 index a06afff6e..000000000 --- a/roles/kubernetes/control-plane/templates/k8s-certs-renew.sh.j2 +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/bash - -echo "## Check Expiration before renewal ##" - -{{ bin_dir }}/kubeadm certs check-expiration - -days_buffer=7 # set a time margin, because we should not renew at the last moment -# systemctl show reports an empty NextElapseUSecRealtime while the timer's unit is -# running, so use the calendar spec to get the next scheduled run. -next_time=$(LC_ALL=C systemd-analyze calendar "{{ auto_renew_certificates_systemd_calendar }}" | grep -oP 'Next elapse:\s*\K.*\S') - -if [ "${next_time}" == "" ] || [ "${next_time}" == "never" ]; then - echo "## Skip expiry comparison due to fail to parse next elapse from systemd calendar,do renewal directly ##" -else - current_time=$(date +%s) - target_time=$(date -d "${next_time} + ${days_buffer} days" +%s) # $next_time - $days_buffer days - expiry_threshold=$(( ${target_time} - ${current_time} )) - # super-admin.conf only exists on the node where "kubeadm init" ran, the other - # control plane nodes see it as missing (residualTime 0). Don't let missing - # certs force a renewal here, kubeadm can't renew them anyway. - expired_certs=$({{ bin_dir }}/kubeadm certs check-expiration -o jsonpath="{range .certificates[?(@.residualTime<${expiry_threshold}.0)]}{.name}={.missing}{\"\n\"}{end}" | sed -n 's/=false$//p') - if [ "${expired_certs}" == "" ];then - echo "## Skip cert renew and K8S container restart, since all residualTimes are beyond threshold ##" - exit 0 - fi - echo "## Certificates to renew: ##" - echo "${expired_certs}" -fi - -echo "## Renewing certificates managed by kubeadm ##" -{{ bin_dir }}/kubeadm certs renew all - -echo "## Restarting control plane pods managed by kubeadm ##" -{% if container_manager == "docker" %} -{{ docker_bin_dir }}/docker ps -af 'name=k8s_POD_(kube-apiserver|kube-controller-manager|kube-scheduler|etcd)-*' -q | /usr/bin/xargs {{ docker_bin_dir }}/docker rm -f -{% else %} -{{ bin_dir }}/crictl pods --namespace kube-system --name 'kube-scheduler-*|kube-controller-manager-*|kube-apiserver-*|etcd-*' -q | /usr/bin/xargs {{ bin_dir }}/crictl rmp -f -{% endif %} - -echo "## Updating /root/.kube/config ##" -cp {{ kube_config_dir }}/admin.conf /root/.kube/config - -echo "## Waiting for apiserver to be up again ##" -until printf "" 2>>/dev/null >>/dev/tcp/127.0.0.1/{{ kube_apiserver_port | default(6443) }}; do sleep 1; done - -echo "## Expiration after renewal ##" -{{ bin_dir }}/kubeadm certs check-expiration diff --git a/roles/reset/tasks/main.yml b/roles/reset/tasks/main.yml index b67f1b105..77bd2371b 100644 --- a/roles/reset/tasks/main.yml +++ b/roles/reset/tasks/main.yml @@ -377,6 +377,7 @@ - "{{ bin_dir }}/crictl" - "{{ bin_dir }}/nerdctl" - "{{ bin_dir }}/netctl" + - "{{ bin_dir }}/k8s-certs-renew.py" - "{{ bin_dir }}/k8s-certs-renew.sh" - /var/lib/cni - /etc/openvswitch