From e87055095c7cfefedfb593309c8576407ef72ffb Mon Sep 17 00:00:00 2001 From: Wander Boessenkool Date: Mon, 14 Oct 2019 14:53:53 +0200 Subject: [PATCH 1/7] Change 'rabbitmqctl status' to a wget | grep - This reduces CPU usage from 250 millis on idle to 25 millis on idle - Default rabbitmq user needs administrator privileges --- .../roles/kubernetes/templates/deployment.yml.j2 | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/installer/roles/kubernetes/templates/deployment.yml.j2 b/installer/roles/kubernetes/templates/deployment.yml.j2 index 8e9f234c20..59b4015a7d 100644 --- a/installer/roles/kubernetes/templates/deployment.yml.j2 +++ b/installer/roles/kubernetes/templates/deployment.yml.j2 @@ -39,7 +39,7 @@ data: [rabbitmq_management,rabbitmq_peer_discovery_k8s]. rabbitmq_definitions.json: | { - "users":[{"name": "{{ rabbitmq_user }}", "password": "{{ rabbitmq_password }}", "tags": ""}], + "users":[{"name": "{{ rabbitmq_user }}", "password": "{{ rabbitmq_password }}", "tags": "administrator"}], "permissions":[ {"user":"{{ rabbitmq_user }}","vhost":"awx","configure":".*","write":".*","read":".*"} ], @@ -269,12 +269,18 @@ spec: containerPort: 5672 livenessProbe: exec: - command: ["rabbitmqctl", "status"] + command: + - /bin/ash + - -c + - "wget -O - --header \"Authorization: Basic {{ ( rabbitmq_user + ':' + rabbitmq_password ) | b64encode }}\" http://localhost:15672/api/healthchecks/node | grep -qF \"{\"status\":\"ok\"}\"" initialDelaySeconds: 30 timeoutSeconds: 10 readinessProbe: exec: - command: ["rabbitmqctl", "status"] + command: + - /bin/ash + - -c + - "wget -O - --header \"Authorization: Basic {{ ( rabbitmq_user + ':' + rabbitmq_password ) | b64encode }}\" http://localhost:15672/api/healthchecks/node | grep -qF \"{\"status\":\"ok\"}\"" initialDelaySeconds: 10 timeoutSeconds: 10 env: From 038fd9271d0b044e9b90448d59a8a4afa605c842 Mon Sep 17 00:00:00 2001 From: Wander Boessenkool Date: Mon, 14 Oct 2019 17:53:28 +0200 Subject: [PATCH 2/7] Properly escape quotes --- installer/roles/kubernetes/templates/deployment.yml.j2 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/installer/roles/kubernetes/templates/deployment.yml.j2 b/installer/roles/kubernetes/templates/deployment.yml.j2 index 59b4015a7d..cb6540a0ff 100644 --- a/installer/roles/kubernetes/templates/deployment.yml.j2 +++ b/installer/roles/kubernetes/templates/deployment.yml.j2 @@ -272,7 +272,7 @@ spec: command: - /bin/ash - -c - - "wget -O - --header \"Authorization: Basic {{ ( rabbitmq_user + ':' + rabbitmq_password ) | b64encode }}\" http://localhost:15672/api/healthchecks/node | grep -qF \"{\"status\":\"ok\"}\"" + - "wget -O - --header \"Authorization: Basic {{ ( rabbitmq_user + ':' + rabbitmq_password ) | b64encode }}\" http://localhost:15672/api/healthchecks/node | grep -qF \"{\\\"status\\\":\\\"ok\\\"}\"" initialDelaySeconds: 30 timeoutSeconds: 10 readinessProbe: @@ -280,7 +280,7 @@ spec: command: - /bin/ash - -c - - "wget -O - --header \"Authorization: Basic {{ ( rabbitmq_user + ':' + rabbitmq_password ) | b64encode }}\" http://localhost:15672/api/healthchecks/node | grep -qF \"{\"status\":\"ok\"}\"" + - "wget -O - --header \"Authorization: Basic {{ ( rabbitmq_user + ':' + rabbitmq_password ) | b64encode }}\" http://localhost:15672/api/healthchecks/node | grep -qF \"{\\\"status\\\":\\\"ok\\\"}\"" initialDelaySeconds: 10 timeoutSeconds: 10 env: From d6134fb194d038cd0ecdc645878edba9402b1668 Mon Sep 17 00:00:00 2001 From: Wander Boessenkool Date: Thu, 17 Oct 2019 21:37:51 +0200 Subject: [PATCH 3/7] Change /bin/ash to /bin/sh as requested by @shanecmd --- installer/roles/kubernetes/templates/deployment.yml.j2 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/installer/roles/kubernetes/templates/deployment.yml.j2 b/installer/roles/kubernetes/templates/deployment.yml.j2 index cb6540a0ff..4162742cee 100644 --- a/installer/roles/kubernetes/templates/deployment.yml.j2 +++ b/installer/roles/kubernetes/templates/deployment.yml.j2 @@ -270,7 +270,7 @@ spec: livenessProbe: exec: command: - - /bin/ash + - /bin/sh - -c - "wget -O - --header \"Authorization: Basic {{ ( rabbitmq_user + ':' + rabbitmq_password ) | b64encode }}\" http://localhost:15672/api/healthchecks/node | grep -qF \"{\\\"status\\\":\\\"ok\\\"}\"" initialDelaySeconds: 30 @@ -278,7 +278,7 @@ spec: readinessProbe: exec: command: - - /bin/ash + - /bin/sh - -c - "wget -O - --header \"Authorization: Basic {{ ( rabbitmq_user + ':' + rabbitmq_password ) | b64encode }}\" http://localhost:15672/api/healthchecks/node | grep -qF \"{\\\"status\\\":\\\"ok\\\"}\"" initialDelaySeconds: 10 From 9ab58e97570e54f93546a14224e4d2efa782217b Mon Sep 17 00:00:00 2001 From: Wander Boessenkool Date: Thu, 17 Oct 2019 22:25:20 +0200 Subject: [PATCH 4/7] Change healthcheck from wget and grep to python with httplib --- .../kubernetes/templates/deployment.yml.j2 | 34 ++++++++++++++++--- 1 file changed, 30 insertions(+), 4 deletions(-) diff --git a/installer/roles/kubernetes/templates/deployment.yml.j2 b/installer/roles/kubernetes/templates/deployment.yml.j2 index 4162742cee..fd09bb7333 100644 --- a/installer/roles/kubernetes/templates/deployment.yml.j2 +++ b/installer/roles/kubernetes/templates/deployment.yml.j2 @@ -270,17 +270,43 @@ spec: livenessProbe: exec: command: - - /bin/sh + - /usr/bin/python - -c - - "wget -O - --header \"Authorization: Basic {{ ( rabbitmq_user + ':' + rabbitmq_password ) | b64encode }}\" http://localhost:15672/api/healthchecks/node | grep -qF \"{\\\"status\\\":\\\"ok\\\"}\"" + - | + import httplib + import sys + conn=httplib.HTTPConnection('localhost:15672') + conn.request('GET', '/api/healthchecks/node', headers={'Authorization': 'Basic {{ ( rabbitmq_user + ':' + rabbitmq_password ) | b64encode }}' }) + r1 = conn.getresponse() + if r1.status != 200: + sys.stderr.write('Received http error %i\n' % (r1.status)) + sys.exit(1) + body = r1.read() + if body != '{"status":"ok"}': + sys.stderr.write('Received body: %s' % body) + sys.exit(2) + sys.exit(0) initialDelaySeconds: 30 timeoutSeconds: 10 readinessProbe: exec: command: - - /bin/sh + - /usr/bin/python - -c - - "wget -O - --header \"Authorization: Basic {{ ( rabbitmq_user + ':' + rabbitmq_password ) | b64encode }}\" http://localhost:15672/api/healthchecks/node | grep -qF \"{\\\"status\\\":\\\"ok\\\"}\"" + - | + import httplib + import sys + conn=httplib.HTTPConnection('localhost:15672') + conn.request('GET', '/api/healthchecks/node', headers={'Authorization': 'Basic {{ ( rabbitmq_user + ':' + rabbitmq_password ) | b64encode }}' }) + r1 = conn.getresponse() + if r1.status != 200: + sys.stderr.write('Received http error %i\n' % (r1.status)) + sys.exit(1) + body = r1.read() + if body != '{"status":"ok"}': + sys.stderr.write('Received body: %s' % body) + sys.exit(2) + sys.exit(0) initialDelaySeconds: 10 timeoutSeconds: 10 env: From 00c9d756e848680381806aff63cffb56431f79ce Mon Sep 17 00:00:00 2001 From: Wander Boessenkool Date: Thu, 17 Oct 2019 23:23:29 +0200 Subject: [PATCH 5/7] Move installtime hardcoded rabbitmq credentials to environment variables for healthcheck --- .../kubernetes/templates/deployment.yml.j2 | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/installer/roles/kubernetes/templates/deployment.yml.j2 b/installer/roles/kubernetes/templates/deployment.yml.j2 index fd09bb7333..91479bbd51 100644 --- a/installer/roles/kubernetes/templates/deployment.yml.j2 +++ b/installer/roles/kubernetes/templates/deployment.yml.j2 @@ -275,8 +275,11 @@ spec: - | import httplib import sys + import os + import base64 + authsecret = base64.b64encode(os.getenv('RABBITMQ_USER') + ':' + os.getenv('RABBITMQ_PASSWORD')) conn=httplib.HTTPConnection('localhost:15672') - conn.request('GET', '/api/healthchecks/node', headers={'Authorization': 'Basic {{ ( rabbitmq_user + ':' + rabbitmq_password ) | b64encode }}' }) + conn.request('GET', '/api/healthchecks/node', headers={'Authorization': 'Basic %s' % authsecret}) r1 = conn.getresponse() if r1.status != 200: sys.stderr.write('Received http error %i\n' % (r1.status)) @@ -296,8 +299,11 @@ spec: - | import httplib import sys + import os + import base64 + authsecret = base64.b64encode(os.getenv('RABBITMQ_USER') + ':' + os.getenv('RABBITMQ_PASSWORD')) conn=httplib.HTTPConnection('localhost:15672') - conn.request('GET', '/api/healthchecks/node', headers={'Authorization': 'Basic {{ ( rabbitmq_user + ':' + rabbitmq_password ) | b64encode }}' }) + conn.request('GET', '/api/healthchecks/node', headers={'Authorization': 'Basic %s' % authsecret}) r1 = conn.getresponse() if r1.status != 200: sys.stderr.write('Received http error %i\n' % (r1.status)) @@ -325,6 +331,13 @@ spec: key: rabbitmq_erlang_cookie - name: K8S_SERVICE_NAME value: "rabbitmq" + - name: RABBITMQ_USER + value: {{ rabbitmq_user }} + - name: RABBITMQ_PASSWORD + valueFrom: + secretKeyRef: + name: "{{ kubernetes_deployment_name }}-secrets" + key: rabbitmq_password volumeMounts: - name: rabbitmq-config mountPath: /etc/rabbitmq From c49e64e62cd228894f7fd892b04eb915c1ddaf66 Mon Sep 17 00:00:00 2001 From: Wander Boessenkool Date: Thu, 17 Oct 2019 23:36:33 +0200 Subject: [PATCH 6/7] Make HTTPConnection import python 2,3 agnostic --- .../roles/kubernetes/templates/deployment.yml.j2 | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/installer/roles/kubernetes/templates/deployment.yml.j2 b/installer/roles/kubernetes/templates/deployment.yml.j2 index 91479bbd51..d71a6ebc54 100644 --- a/installer/roles/kubernetes/templates/deployment.yml.j2 +++ b/installer/roles/kubernetes/templates/deployment.yml.j2 @@ -273,12 +273,15 @@ spec: - /usr/bin/python - -c - | - import httplib + try: + from http.client import HTTPConnection + except ImportError: + from httplib import HTTPConnection import sys import os import base64 authsecret = base64.b64encode(os.getenv('RABBITMQ_USER') + ':' + os.getenv('RABBITMQ_PASSWORD')) - conn=httplib.HTTPConnection('localhost:15672') + conn=HTTPConnection('localhost:15672') conn.request('GET', '/api/healthchecks/node', headers={'Authorization': 'Basic %s' % authsecret}) r1 = conn.getresponse() if r1.status != 200: @@ -297,12 +300,15 @@ spec: - /usr/bin/python - -c - | - import httplib + try: + from http.client import HTTPConnection + except ImportError: + from httplib import HTTPConnection import sys import os import base64 authsecret = base64.b64encode(os.getenv('RABBITMQ_USER') + ':' + os.getenv('RABBITMQ_PASSWORD')) - conn=httplib.HTTPConnection('localhost:15672') + conn=HTTPConnection('localhost:15672') conn.request('GET', '/api/healthchecks/node', headers={'Authorization': 'Basic %s' % authsecret}) r1 = conn.getresponse() if r1.status != 200: From 8ecc1f37f08e5b8e6ffddedbe31162de4b08386e Mon Sep 17 00:00:00 2001 From: Wander Boessenkool Date: Fri, 18 Oct 2019 10:15:21 +0200 Subject: [PATCH 7/7] Move python healthcheck script from probes to configMap --- .../kubernetes/templates/deployment.yml.j2 | 83 +++++++++---------- 1 file changed, 39 insertions(+), 44 deletions(-) diff --git a/installer/roles/kubernetes/templates/deployment.yml.j2 b/installer/roles/kubernetes/templates/deployment.yml.j2 index d71a6ebc54..383a3e0a8a 100644 --- a/installer/roles/kubernetes/templates/deployment.yml.j2 +++ b/installer/roles/kubernetes/templates/deployment.yml.j2 @@ -270,55 +270,13 @@ spec: livenessProbe: exec: command: - - /usr/bin/python - - -c - - | - try: - from http.client import HTTPConnection - except ImportError: - from httplib import HTTPConnection - import sys - import os - import base64 - authsecret = base64.b64encode(os.getenv('RABBITMQ_USER') + ':' + os.getenv('RABBITMQ_PASSWORD')) - conn=HTTPConnection('localhost:15672') - conn.request('GET', '/api/healthchecks/node', headers={'Authorization': 'Basic %s' % authsecret}) - r1 = conn.getresponse() - if r1.status != 200: - sys.stderr.write('Received http error %i\n' % (r1.status)) - sys.exit(1) - body = r1.read() - if body != '{"status":"ok"}': - sys.stderr.write('Received body: %s' % body) - sys.exit(2) - sys.exit(0) + - /usr/local/bin/healthchecks/rabbit_health_node.py initialDelaySeconds: 30 timeoutSeconds: 10 readinessProbe: exec: command: - - /usr/bin/python - - -c - - | - try: - from http.client import HTTPConnection - except ImportError: - from httplib import HTTPConnection - import sys - import os - import base64 - authsecret = base64.b64encode(os.getenv('RABBITMQ_USER') + ':' + os.getenv('RABBITMQ_PASSWORD')) - conn=HTTPConnection('localhost:15672') - conn.request('GET', '/api/healthchecks/node', headers={'Authorization': 'Basic %s' % authsecret}) - r1 = conn.getresponse() - if r1.status != 200: - sys.stderr.write('Received http error %i\n' % (r1.status)) - sys.exit(1) - body = r1.read() - if body != '{"status":"ok"}': - sys.stderr.write('Received body: %s' % body) - sys.exit(2) - sys.exit(0) + - /usr/local/bin/healthchecks/rabbit_health_node.py initialDelaySeconds: 10 timeoutSeconds: 10 env: @@ -347,6 +305,8 @@ spec: volumeMounts: - name: rabbitmq-config mountPath: /etc/rabbitmq + - name: rabbitmq-healthchecks + mountPath: /usr/local/bin/healthchecks resources: requests: memory: "{{ rabbitmq_mem_request }}Gi" @@ -438,6 +398,41 @@ spec: path: enabled_plugins - key: rabbitmq_definitions.json path: rabbitmq_definitions.json + - name: rabbitmq-healthchecks + configMap: + name: {{ kubernetes_deployment_name }}-healthchecks + items: + - key: rabbit_health_node.py + path: rabbit_health_node.py + defaultMode: 0755 +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ kubernetes_deployment_name }}-healthchecks + namespace: {{ kubernetes_namespace }} +data: + rabbit_health_node.py: | + #!/usr/bin/env python + try: + from http.client import HTTPConnection + except ImportError: + from httplib import HTTPConnection + import sys + import os + import base64 + authsecret = base64.b64encode(os.getenv('RABBITMQ_USER') + ':' + os.getenv('RABBITMQ_PASSWORD')) + conn=HTTPConnection('localhost:15672') + conn.request('GET', '/api/healthchecks/node', headers={'Authorization': 'Basic %s' % authsecret}) + r1 = conn.getresponse() + if r1.status != 200: + sys.stderr.write('Received http error %i\n' % (r1.status)) + sys.exit(1) + body = r1.read() + if body != '{"status":"ok"}': + sys.stderr.write('Received body: %s' % body) + sys.exit(2) + sys.exit(0) --- apiVersion: v1 kind: Service