Initial migration of rabbitmq -> redis for k8s installs

This commit is contained in:
Shane McDonald 2020-02-13 13:16:25 -05:00 committed by Ryan Petrello
parent e94bb44082
commit 45ce6d794e
No known key found for this signature in database
GPG Key ID: F2AA5F2122351777
14 changed files with 84 additions and 378 deletions

View File

@ -38,11 +38,12 @@ def unwrap_broadcast_msg(payload: dict):
def get_broadcast_hosts():
Instance = apps.get_model('main', 'Instance')
return [h[0] for h in Instance.objects.filter(rampart_groups__controller__isnull=True)
.exclude(hostname=Instance.objects.me().hostname)
.order_by('hostname')
.values_list('hostname')
.distinct()]
instances = Instance.objects.filter(rampart_groups__controller__isnull=True) \
.exclude(hostname=Instance.objects.me().hostname) \
.order_by('hostname') \
.values('hostname', 'ip_address') \
.distinct()
return [i['ip_address'] or i['hostname'] for i in instances]
def get_local_host():

View File

@ -3,6 +3,7 @@
import sys
import logging
import os
from django.db import models
from django.conf import settings
@ -114,7 +115,7 @@ class InstanceManager(models.Manager):
return node[0]
raise RuntimeError("No instance found with the current cluster host id")
def register(self, uuid=None, hostname=None):
def register(self, uuid=None, hostname=None, ip_address=None):
if not uuid:
uuid = settings.SYSTEM_UUID
if not hostname:
@ -122,13 +123,23 @@ class InstanceManager(models.Manager):
with advisory_lock('instance_registration_%s' % hostname):
instance = self.filter(hostname=hostname)
if instance.exists():
return (False, instance[0])
instance = self.create(uuid=uuid, hostname=hostname, capacity=0)
instance = instance.get()
if instance.ip_address != ip_address:
instance.ip_address = ip_address
instance.save()
return (True, instance)
else:
return (False, instance)
instance = self.create(uuid=uuid,
hostname=hostname,
ip_address=ip_address,
capacity=0)
return (True, instance)
def get_or_register(self):
if settings.AWX_AUTO_DEPROVISION_INSTANCES:
return self.register()
pod_ip = os.environ.get('MY_POD_IP')
return self.register(ip_address=pod_ip)
else:
return (False, self.me())

View File

@ -0,0 +1,18 @@
# Generated by Django 2.2.8 on 2020-02-12 17:55
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('main', '0108_v370_unifiedjob_dependencies_processed'),
]
operations = [
migrations.AddField(
model_name='instance',
name='ip_address',
field=models.CharField(blank=True, default=None, max_length=50, null=True, unique=True),
),
]

View File

@ -53,6 +53,13 @@ class Instance(HasPolicyEditsMixin, BaseModel):
uuid = models.CharField(max_length=40)
hostname = models.CharField(max_length=250, unique=True)
ip_address = models.CharField(
blank=True,
null=True,
default=None,
max_length=50,
unique=True,
)
created = models.DateTimeField(auto_now_add=True)
modified = models.DateTimeField(auto_now=True)
last_isolated_check = models.DateTimeField(

View File

@ -38,7 +38,7 @@ dockerhub_base=ansible
# kubernetes_ingress_tls_secret=awx-cert
# Kubernetes and Openshift Install Resource Requests
# These are the request and limit values for a pod's container for task/web/rabbitmq/memcached/management.
# These are the request and limit values for a pod's container for task/web/redis/memcached/management.
# The total amount of requested resources for a pod is the sum of all
# resources requested by all containers in the pod
# A cpu_request of 1500 is 1.5 cores for the container to start out with.
@ -52,8 +52,8 @@ dockerhub_base=ansible
# task_mem_limit=4
# web_cpu_limit=1000
# web_mem_limit=2
# rabbitmq_cpu_limit=1000
# rabbitmq_mem_limit=3
# redis_cpu_limit=1000
# redis_mem_limit=3
# memcached_cpu_limit=1000
# memcached_mem_limit=2
# management_cpu_limit=2000

View File

@ -6,10 +6,6 @@ admin_user: 'admin'
admin_email: 'root@localhost'
admin_password: ''
rabbitmq_user: 'awx'
rabbitmq_password: ''
rabbitmq_erlang_cookie: ''
kubernetes_base_path: "{{ local_base_config_path|default('/tmp') }}/{{ kubernetes_deployment_name }}-config"
kubernetes_task_version: "{{ tower_package_version | default(dockerhub_version) }}"
@ -24,16 +20,18 @@ web_cpu_request: 500
task_mem_request: 2
task_cpu_request: 1500
rabbitmq_mem_request: 2
rabbitmq_cpu_request: 500
redis_mem_request: 2
redis_cpu_request: 500
kubernetes_redis_image: "redis"
kubernetes_redis_image_tag: "latest"
kubernetes_redis_hostname: "localhost"
kubernetes_redis_port: "6379"
memcached_hostname: localhost
memcached_mem_request: 1
memcached_cpu_request: 500
kubernetes_rabbitmq_version: "3.7.21"
kubernetes_rabbitmq_image: "ansible/awx_rabbitmq"
kubernetes_memcached_version: "latest"
kubernetes_memcached_image: "memcached"
@ -56,6 +54,5 @@ custom_venvs_path: "/opt/custom-venvs"
custom_venvs_python: "python2"
ca_trust_bundle: "/etc/pki/tls/certs/ca-bundle.crt"
rabbitmq_use_ssl: false
container_groups_image: "ansible/ansible-runner"

View File

@ -194,10 +194,6 @@
when: kubernetes_web_image is not defined
when: docker_registry is defined
- name: Generate SSL certificates for RabbitMQ, if needed
include_tasks: ssl_cert_gen.yml
when: "rabbitmq_use_ssl|default(False)|bool"
- name: Determine StatefulSet api version
set_fact:
kubernetes_statefulset_api_version: "{{ 'apps/v1' if kube_api_version is version('1.9', '>=') else 'apps/v1beta1' }}"

View File

@ -1,60 +0,0 @@
---
- name: Create temporary directory
tempfile:
state: directory
prefix: "tower-install-rmq-certs"
register: rmq_cert_tempdir
notify: remove-rmq_cert_tempdir
- name: Generate CA private key
openssl_privatekey:
path: '{{ rmq_cert_tempdir.path }}/ca.key'
mode: "0600"
- name: Generate CA CSR
openssl_csr:
path: '{{ rmq_cert_tempdir.path }}/ca.csr'
privatekey_path: '{{ rmq_cert_tempdir.path }}/ca.key'
common_name: 'rabbitmq-ca'
basic_constraints: 'CA:TRUE'
mode: "0600"
- name: Generate CA certificate
openssl_certificate:
path: '{{ rmq_cert_tempdir.path }}/ca.crt'
csr_path: '{{ rmq_cert_tempdir.path }}/ca.csr'
privatekey_path: '{{ rmq_cert_tempdir.path }}/ca.key'
provider: selfsigned
selfsigned_not_after: "+36524d"
mode: "0600"
- name: Generate server private key
openssl_privatekey:
path: '{{ rmq_cert_tempdir.path }}/server.key'
mode: "0600"
- name: Generate server CSR
openssl_csr:
path: '{{ rmq_cert_tempdir.path }}/server.csr'
privatekey_path: '{{ rmq_cert_tempdir.path }}/server.key'
common_name: 'rabbitmq-server'
mode: "0600"
- name: Generate server certificate
openssl_certificate:
path: "{{ rmq_cert_tempdir.path }}/server.crt"
csr_path: "{{ rmq_cert_tempdir.path }}/server.csr"
privatekey_path: "{{ rmq_cert_tempdir.path }}/server.key"
provider: ownca
ownca_path: "{{ rmq_cert_tempdir.path }}/ca.crt"
ownca_privatekey_path: "{{ rmq_cert_tempdir.path }}/ca.key"
ownca_not_after: "+36500d"
mode: "0600"
- name: Create combined certificate
assemble:
src: "{{ rmq_cert_tempdir.path }}"
regexp: "server.crt|server.key"
dest: "{{ rmq_cert_tempdir.path }}/server-combined.pem"
mode: "0600"

View File

@ -205,3 +205,5 @@ data:
USE_X_FORWARDED_PORT = True
AWX_CONTAINER_GROUP_DEFAULT_IMAGE = "{{ container_groups_image }}"
BROADCAST_WEBSOCKETS_PORT = 8052
BROADCAST_WEBSOCKETS_PROTOCOL = 'http'

View File

@ -12,14 +12,12 @@ DATABASES = {
},
}
}
BROKER_URL = 'amqp://{}:{}@{}:{}/{}'.format(
"{{ rabbitmq_user }}",
"{{ rabbitmq_password }}",
"localhost",
"5672",
"awx")
BROKER_URL = 'redis://{}:{}/'.format(
"{{ kubernetes_redis_hostname }}",
"{{ kubernetes_redis_port }}",)
CHANNEL_LAYERS = {
'default': {'BACKEND': 'asgi_amqp.AMQPChannelLayer',
'ROUTING': 'awx.main.routing.channel_routing',
'CONFIG': {'url': BROKER_URL}}
}
'default': {'BACKEND': 'awx.main.channels.RedisGroupBroadcastChannelLayer',
'CONFIG': {'hosts': [("{{ kubernetes_redis_hostname }}", {{ kubernetes_redis_port|int }})]}}
}

View File

@ -15,131 +15,6 @@ imagePullSecrets:
- name: "{{ kubernetes_image_pull_secrets }}"
{% endif %}
---
kind: Service
apiVersion: v1
metadata:
namespace: {{ kubernetes_namespace }}
name: rabbitmq
labels:
app: {{ kubernetes_deployment_name }}
type: LoadBalancer
spec:
type: NodePort
ports:
- name: http
protocol: TCP
port: 15672
targetPort: 15672
- name: amqp
protocol: TCP
port: 5672
targetPort: 5672
selector:
app: {{ kubernetes_deployment_name }}
---
apiVersion: v1
kind: ConfigMap
metadata:
name: rabbitmq-config
namespace: {{ kubernetes_namespace }}
data:
enabled_plugins: |
[rabbitmq_management,rabbitmq_peer_discovery_k8s].
rabbitmq_definitions.json: |
{
"users":[{"name": "{{ rabbitmq_user }}", "password": "{{ rabbitmq_password }}", "tags": "administrator"}],
"permissions":[
{"user":"{{ rabbitmq_user }}","vhost":"awx","configure":".*","write":".*","read":".*"}
],
"vhosts":[{"name":"awx"}],
"policies":[
{"vhost":"awx","name":"ha-all","pattern":".*","definition":{"ha-mode":"all","ha-sync-mode":"automatic"}}
]
}
rabbitmq.conf: |
## Clustering
management.load_definitions = /etc/rabbitmq/rabbitmq_definitions.json
cluster_formation.peer_discovery_backend = rabbit_peer_discovery_k8s
cluster_formation.k8s.host = kubernetes.default.svc
cluster_formation.k8s.address_type = ip
cluster_formation.node_cleanup.interval = 10
cluster_formation.node_cleanup.only_log_warning = false
cluster_partition_handling = autoheal
## queue master locator
queue_master_locator=min-masters
## enable guest user
loopback_users.guest = false
{% if rabbitmq_use_ssl|default(False)|bool %}
ssl_options.cacertfile=/etc/pki/rabbitmq/ca.crt
ssl_options.certfile=/etc/pki/rabbitmq/server-combined.pem
ssl_options.verify=verify_peer
{% endif %}
rabbitmq-env.conf: |
NODENAME=${RABBITMQ_NODENAME}
USE_LONGNAME=true
{% if rabbitmq_use_ssl|default(False)|bool %}
ERL_SSL_PATH=$(erl -eval 'io:format("~p", [code:lib_dir(ssl, ebin)]),halt().' -noshell)
SSL_ADDITIONAL_ERL_ARGS="-pa '$ERL_SSL_PATH' -proto_dist inet_tls -ssl_dist_opt server_certfile /etc/pki/rabbitmq/server-combined.pem -ssl_dist_opt server_secure_renegotiate true client_secure_renegotiate true"
SERVER_ADDITIONAL_ERL_ARGS="$SERVER_ADDITIONAL_ERL_ARGS $SSL_ADDITIONAL_ERL_ARGS"
CTL_ERL_ARGS="$SSL_ADDITIONAL_ERL_ARGS"
{% endif %}
{% if kubernetes_context is defined %}
---
kind: Role
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
name: endpoint-reader
namespace: {{ kubernetes_namespace }}
rules:
- apiGroups: [""]
resources: ["endpoints"]
verbs: ["get"]
---
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
name: endpoint-reader
namespace: {{ kubernetes_namespace }}
subjects:
- kind: ServiceAccount
name: awx
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: endpoint-reader
{% endif %}
{% if openshift_host is defined %}
---
kind: Role
apiVersion: v1
metadata:
name: endpoint-reader
namespace: {{ kubernetes_namespace }}
rules:
- apiGroups: [""]
resources: ["endpoints"]
verbs: ["get"]
---
kind: RoleBinding
apiVersion: v1
metadata:
name: endpoint-reader
namespace: {{ kubernetes_namespace }}
roleRef:
name: endpoint-reader
namespace: {{ kubernetes_namespace }}
subjects:
- kind: ServiceAccount
name: awx
namespace: {{ kubernetes_namespace }}
userNames:
- system:serviceaccount:{{ kubernetes_namespace }}:awx
{% endif %}
---
apiVersion: {{ kubernetes_statefulset_api_version }}
kind: StatefulSet
@ -165,7 +40,6 @@ spec:
service: django
app: {{ kubernetes_deployment_name }}
spec:
serviceAccountName: awx
terminationGracePeriodSeconds: 10
{% if custom_venvs is defined %}
{% set trusted_hosts = "" %}
@ -266,7 +140,7 @@ spec:
{% if web_cpu_limit is defined %}
cpu: "{{ web_cpu_limit }}m"
{% endif %}
- name: {{ kubernetes_deployment_name }}-celery
- name: {{ kubernetes_deployment_name }}-task
securityContext:
privileged: true
image: "{{ kubernetes_task_image }}:{{ kubernetes_task_version }}"
@ -303,6 +177,10 @@ spec:
valueFrom:
fieldRef:
fieldPath: metadata.uid
- name: MY_POD_IP
valueFrom:
fieldRef:
fieldPath: status.podIP
resources:
requests:
memory: "{{ task_mem_request }}Gi"
@ -316,72 +194,25 @@ spec:
{% if task_cpu_limit is defined %}
cpu: "{{ task_cpu_limit }}m"
{% endif %}
- name: {{ kubernetes_deployment_name }}-rabbit
image: "{{ kubernetes_rabbitmq_image }}:{{ kubernetes_rabbitmq_version }}"
- name: {{ kubernetes_deployment_name }}-redis
image: {{ kubernetes_redis_image }}:{{ kubernetes_redis_image_tag }}
imagePullPolicy: Always
ports:
- name: http
- name: redis
protocol: TCP
containerPort: 15672
- name: amqp
protocol: TCP
containerPort: 5672
livenessProbe:
exec:
command:
- /usr/local/bin/healthchecks/rabbit_health_node.py
initialDelaySeconds: 30
timeoutSeconds: 10
readinessProbe:
exec:
command:
- /usr/local/bin/healthchecks/rabbit_health_node.py
initialDelaySeconds: 10
timeoutSeconds: 10
env:
- name: MY_POD_IP
valueFrom:
fieldRef:
fieldPath: status.podIP
- name: RABBITMQ_USE_LONGNAME
value: "true"
- name: RABBITMQ_NODENAME
value: "rabbit@$(MY_POD_IP)"
- name: RABBITMQ_ERLANG_COOKIE
valueFrom:
secretKeyRef:
name: "{{ kubernetes_deployment_name }}-secrets"
key: rabbitmq_erlang_cookie
- name: K8S_SERVICE_NAME
value: "rabbitmq"
- name: RABBITMQ_USER
value: {{ rabbitmq_user }}
- name: RABBITMQ_PASSWORD
valueFrom:
secretKeyRef:
name: "{{ kubernetes_deployment_name }}-secrets"
key: rabbitmq_password
volumeMounts:
- name: rabbitmq-config
mountPath: /etc/rabbitmq
- name: rabbitmq-healthchecks
mountPath: /usr/local/bin/healthchecks
{% if rabbitmq_use_ssl|default(False)|bool %}
- name: "{{ kubernetes_deployment_name }}-rabbitmq-certs-vol"
mountPath: /etc/pki/rabbitmq
{% endif %}
containerPort: 6379
resources:
requests:
memory: "{{ rabbitmq_mem_request }}Gi"
cpu: "{{ rabbitmq_cpu_request }}m"
{% if rabbitmq_mem_limit is defined or rabbitmq_cpu_limit is defined %}
memory: "{{ redis_mem_request }}Gi"
cpu: "{{ redis_cpu_request }}m"
{% if redis_mem_limit is defined or redis_cpu_limit is defined %}
limits:
{% endif %}
{% if rabbitmq_mem_limit is defined %}
memory: "{{ rabbitmq_mem_limit }}Gi"
{% if redis_mem_limit is defined %}
memory: "{{ redis_mem_limit }}Gi"
{% endif %}
{% if rabbitmq_cpu_limit is defined %}
cpu: "{{ rabbitmq_cpu_limit }}m"
{% if redis_cpu_limit is defined %}
cpu: "{{ redis_cpu_limit }}m"
{% endif %}
- name: {{ kubernetes_deployment_name }}-memcached
image: "{{ kubernetes_memcached_image }}:{{ kubernetes_memcached_version }}"
@ -458,68 +289,6 @@ spec:
- key: secret_key
path: SECRET_KEY
- name: rabbitmq-config
configMap:
name: rabbitmq-config
items:
- key: rabbitmq.conf
path: rabbitmq.conf
- key: enabled_plugins
path: enabled_plugins
- key: rabbitmq_definitions.json
path: rabbitmq_definitions.json
- key: rabbitmq-env.conf
path: rabbitmq-env.conf
{% if rabbitmq_use_ssl|default(False)|bool %}
- name: "{{ kubernetes_deployment_name }}-rabbitmq-certs-vol"
secret:
secretName: "{{ kubernetes_deployment_name }}-rabbitmq-certs"
items:
- key: rabbitmq_ssl_cert
path: 'server.crt'
- key: rabbitmq_ssl_key
path: 'server.key'
- key: rabbitmq_ssl_cacert
path: 'ca.crt'
- key: rabbitmq_ssl_combined
path: 'server-combined.pem'
{% endif %}
- name: rabbitmq-healthchecks
configMap:
name: {{ kubernetes_deployment_name }}-healthchecks
items:
- key: rabbit_health_node.py
path: rabbit_health_node.py
defaultMode: 0755
---
apiVersion: v1
kind: ConfigMap
metadata:
name: {{ kubernetes_deployment_name }}-healthchecks
namespace: {{ kubernetes_namespace }}
data:
rabbit_health_node.py: |
#!/usr/bin/env python
try:
from http.client import HTTPConnection
except ImportError:
from httplib import HTTPConnection
import sys
import os
import base64
authsecret = base64.b64encode(os.getenv('RABBITMQ_USER') + ':' + os.getenv('RABBITMQ_PASSWORD'))
conn=HTTPConnection('localhost:15672')
conn.request('GET', '/api/healthchecks/node', headers={'Authorization': 'Basic %s' % authsecret})
r1 = conn.getresponse()
if r1.status != 200:
sys.stderr.write('Received http error %i\\n' % (r1.status))
sys.exit(1)
body = r1.read()
if body != '{"status":"ok"}':
sys.stderr.write('Received body: %s' % body)
sys.exit(2)
sys.exit(0)
---
apiVersion: v1
kind: Service
@ -536,22 +305,7 @@ spec:
targetPort: 8052
selector:
name: {{ kubernetes_deployment_name }}-web-deploy
---
apiVersion: v1
kind: Service
metadata:
name: {{ kubernetes_deployment_name }}-rmq-mgmt
namespace: {{ kubernetes_namespace }}
labels:
name: {{ kubernetes_deployment_name }}-rmq-mgmt
spec:
type: ClusterIP
ports:
- name: rmqmgmt
port: 15672
targetPort: 15672
selector:
name: {{ kubernetes_deployment_name }}-web-deploy
{% if kubernetes_context is defined %}
---
apiVersion: extensions/v1beta1

View File

@ -5,5 +5,3 @@ DATABASE_PORT={{ pg_port|default('5432') }}
DATABASE_PASSWORD={{ pg_password | quote }}
MEMCACHED_HOST={{ memcached_hostname|default('localhost') }}
MEMCACHED_PORT={{ memcached_port|default('11211') }}
RABBITMQ_HOST={{ rabbitmq_hostname|default('localhost') }}
RABBITMQ_PORT={{ rabbitmq_port|default('5672') }}

View File

@ -12,6 +12,7 @@ spec:
containers:
- name: ansible-tower-management
image: "{{ kubernetes_task_image }}:{{ kubernetes_task_version }}"
imagePullPolicy: Always
command: ["sleep", "infinity"]
volumeMounts:
- name: {{ kubernetes_deployment_name }}-application-config

View File

@ -7,22 +7,5 @@ metadata:
type: Opaque
data:
secret_key: "{{ secret_key | b64encode }}"
rabbitmq_password: "{{ rabbitmq_password | b64encode }}"
rabbitmq_erlang_cookie: "{{ rabbitmq_erlang_cookie | b64encode }}"
credentials_py: "{{ lookup('template', 'credentials.py.j2') | b64encode }}"
environment_sh: "{{ lookup('template', 'environment.sh.j2') | b64encode }}"
{% if rabbitmq_use_ssl|default(False)|bool %}
---
apiVersion: v1
kind: Secret
metadata:
namespace: {{ kubernetes_namespace }}
name: "{{ kubernetes_deployment_name }}-rabbitmq-certs"
type: Opaque
data:
rabbitmq_ssl_cert: "{{ lookup('file', rmq_cert_tempdir.path + '/server.crt') | b64encode }}"
rabbitmq_ssl_key: "{{ lookup('file', rmq_cert_tempdir.path + '/server.key') | b64encode }}"
rabbitmq_ssl_cacert: "{{ lookup('file', rmq_cert_tempdir.path + '/ca.crt') | b64encode }}"
rabbitmq_ssl_combined: "{{ lookup('file', rmq_cert_tempdir.path + '/server-combined.pem') | b64encode }}"
{% endif %}