From b0cf4de07224902992de5ad9bb957cd94855f759 Mon Sep 17 00:00:00 2001 From: Matthew Jones Date: Wed, 14 Mar 2018 13:03:55 -0400 Subject: [PATCH] Implement container-cluster aware capacity determination * Added two settings values for declaring absolute cpu and memory capacity that will be picked up by the capacity utility methods * installer inventory variables for controlling the amount of cpu and memory container requests/limits for the awx task containers * Added fixed values for cpu and memory container requests for other containers * configmap uses the declared inventory variables to define the capacity inputs that will be used by AWX to correspond to the same inputs for requests/limits on the deployment. --- awx/main/utils/common.py | 18 ++++++++ installer/inventory | 42 ++++++++++++------- .../kubernetes/templates/configmap.yml.j2 | 3 ++ .../kubernetes/templates/deployment.yml.j2 | 19 +++++++++ .../openshift/templates/configmap.yml.j2 | 3 ++ .../openshift/templates/deployment.yml.j2 | 19 +++++++++ 6 files changed, 88 insertions(+), 16 deletions(-) diff --git a/awx/main/utils/common.py b/awx/main/utils/common.py index 10865d3cd7..ba3413a133 100644 --- a/awx/main/utils/common.py +++ b/awx/main/utils/common.py @@ -646,6 +646,15 @@ def get_cpu_capacity(): from django.conf import settings settings_forkcpu = getattr(settings, 'SYSTEM_TASK_FORKS_CPU', None) env_forkcpu = os.getenv('SYSTEM_TASK_FORKS_CPU', None) + + settings_abscpu = getattr(settings, 'SYSTEM_TASK_ABS_CPU', None) + env_abscpu = os.getenv('SYSTEM_TASK_ABS_CPU', None) + + if env_abscpu is not None: + return 0, int(env_abscpu) + elif settings_abscpu is not None: + return 0, int(settings_abscpu) + cpu = psutil.cpu_count() if env_forkcpu: @@ -661,6 +670,15 @@ def get_mem_capacity(): from django.conf import settings settings_forkmem = getattr(settings, 'SYSTEM_TASK_FORKS_MEM', None) env_forkmem = os.getenv('SYSTEM_TASK_FORKS_MEM', None) + + settings_absmem = getattr(settings, 'SYSTEM_TASK_ABS_MEM', None) + env_absmem = os.getenv('SYSTEM_TASK_ABS_MEM', None) + + if env_absmem is not None: + return 0, int(env_absmem) + elif settings_absmem is not None: + return 0, int(settings_absmem) + if env_forkmem: forkmem = int(env_forkmem) elif settings_forkmem: diff --git a/installer/inventory b/installer/inventory index c181516ae4..7053a1ed6b 100644 --- a/installer/inventory +++ b/installer/inventory @@ -10,16 +10,6 @@ dockerhub_base=ansible dockerhub_version=latest rabbitmq_version=3.6.14 -# This will create or update a default admin (superuser) account in AWX, if not provided -# then these default values are used -# default_admin_user=admin -# default_admin_password=password - -# AWX Secret key -# It's *very* important that this stay the same between upgrades or you will lose the ability to decrypt -# your credentials -awx_secret_key=awxsecret - # Openshift Install # Will need to set -e openshift_password=developer -e docker_registry_password=$(oc whoami -t) # openshift_host=127.0.0.1:8443 @@ -31,10 +21,26 @@ awx_secret_key=awxsecret # kubernetes_context=test-cluster # awx_kubernetes_namespace=awx +# Kubernetes and Openshift Install Resource Requests +# This is the request value for a pod's "task" container, which is the container +# used to run jobs. The other containers have a fixed resource request. The total amount +# of requested resources for a pod is the sum of all resources requested by all containers +# in the pod +# A cpu_request of 1500 is 1.5 cores for the task container +# A mem_request of 2 is for 2 gigabytes of memory for the task container +# awx_task_cpu_request=1500 +# awx_task_mem_request=2 + # Common Docker parameters postgres_data_dir=/tmp/pgdocker host_port=80 +# Docker Compose Install +# use_docker_compose=false +# The docker_compose.yml file will be created in this directory +# The name of the directory (here "awx") will be the prefix of the docker containers +docker_compose_dir=/var/lib/awx + # Required for Openshift when building the image on your own # Optional for Openshift if using Dockerhub or another prebuilt registry # Required for Standalone Docker Install if building the image on your own @@ -46,12 +52,6 @@ host_port=80 # docker_registry_repository=awx # docker_registry_username=developer -# Docker Compose Install -# use_docker_compose=false -# The docker_compose.yml file will be created in this directory -# The name of the directory (here "awx") will be the prefix of the docker containers -docker_compose_dir=/var/lib/awx - # Docker_image will not attempt to push to remote if the image already exists locally # Set this to true to delete images from docker on the build host so that they are pushed to the remote repository @@ -76,6 +76,16 @@ pg_port=5432 # installer/image_build/files/Dockerfile.sdist # use_container_for_build=true +# This will create or update a default admin (superuser) account in AWX, if not provided +# then these default values are used +# default_admin_user=admin +# default_admin_password=password + +# AWX Secret key +# It's *very* important that this stay the same between upgrades or you will lose the ability to decrypt +# your credentials +awx_secret_key=awxsecret + # Build AWX with official logos # Requires cloning awx-logos repo into the project root. # Review the trademark guidelines at https://github.com/ansible/awx-logos/blob/master/TRADEMARKS.md diff --git a/installer/kubernetes/templates/configmap.yml.j2 b/installer/kubernetes/templates/configmap.yml.j2 index bfcf555c63..5c59f46da6 100644 --- a/installer/kubernetes/templates/configmap.yml.j2 +++ b/installer/kubernetes/templates/configmap.yml.j2 @@ -16,6 +16,9 @@ data: # Automatically deprovision pods that go offline AWX_AUTO_DEPROVISION_INSTANCES = True + SYSTEM_TASK_ABS_CPU = {{ ((awx_task_cpu_request|default(1500) / 1000) * 4)|int }} + SYSTEM_TASK_ABS_MEM = {{ ((awx_task_mem_request|default(2) * 1024) / 100)|int }} + #Autoprovisioning should replace this CLUSTER_HOST_ID = socket.gethostname() SYSTEM_UUID = '00000000-0000-0000-0000-000000000000' diff --git a/installer/kubernetes/templates/deployment.yml.j2 b/installer/kubernetes/templates/deployment.yml.j2 index 00db600273..06cd62e398 100644 --- a/installer/kubernetes/templates/deployment.yml.j2 +++ b/installer/kubernetes/templates/deployment.yml.j2 @@ -21,6 +21,10 @@ spec: volumeMounts: - mountPath: /etc/tower name: awx-application-config + resources: + requests: + memory: "1Gi" + cpu: "500m" - name: awx-celery image: {{ awx_task_kubernetes_image }} imagePullPolicy: Always @@ -46,6 +50,13 @@ spec: value: {{ default_admin_user|default('admin') }} - name: AWX_ADMIN_PASSWORD value: {{ default_admin_password|default('password') }} + resources: + requests: + memory: "{{ awx_task_cpu_request|default('2') }}Gi" + cpu: "{{ awx_task_mem_request|default('1500') }}m" + limit: + memory: "{{ awx_task_cpu_request|default('2') }}Gi" + cpu: "{{ awx_task_mem_request|default('1500') }}m" - name: awx-rabbit image: ansible/awx_rabbitmq:{{ rabbitmq_version }} imagePullPolicy: Always @@ -83,8 +94,16 @@ spec: value: "awx" - name: RABBITMQ_CONFIG_FILE value: "/etc/rabbitmq/rabbitmq" + resources: + requests: + memory: "2Gi" + cpu: "500m" - name: awx-memcached image: memcached + resources: + requests: + memory: "1Gi" + cpu: "500m" volumes: - name: awx-application-config configMap: diff --git a/installer/openshift/templates/configmap.yml.j2 b/installer/openshift/templates/configmap.yml.j2 index b8e5f42a83..c53f3fae85 100644 --- a/installer/openshift/templates/configmap.yml.j2 +++ b/installer/openshift/templates/configmap.yml.j2 @@ -16,6 +16,9 @@ data: # Automatically deprovision pods that go offline AWX_AUTO_DEPROVISION_INSTANCES = True + SYSTEM_TASK_ABS_CPU = {{ ((awx_task_cpu_request|default(1500) / 1000) * 4)|int }} + SYSTEM_TASK_ABS_MEM = {{ ((awx_task_mem_request|default(2) * 1024) / 100)|int }} + #Autoprovisioning should replace this CLUSTER_HOST_ID = socket.gethostname() SYSTEM_UUID = '00000000-0000-0000-0000-000000000000' diff --git a/installer/openshift/templates/deployment.yml.j2 b/installer/openshift/templates/deployment.yml.j2 index 93a9c6cb8c..81e0c46f04 100644 --- a/installer/openshift/templates/deployment.yml.j2 +++ b/installer/openshift/templates/deployment.yml.j2 @@ -21,6 +21,10 @@ spec: volumeMounts: - mountPath: /etc/tower name: awx-application-config + resources: + requests: + memory: "1Gi" + cpu: "500m" - name: awx-celery image: {{ awx_task_openshift_image }} imagePullPolicy: Always @@ -46,6 +50,13 @@ spec: value: {{ default_admin_user|default('admin') }} - name: AWX_ADMIN_PASSWORD value: {{ default_admin_password|default('password') }} + resources: + requests: + memory: "{{ awx_task_cpu_request|default('2') }}Gi" + cpu: "{{ awx_task_mem_request|default('1500') }}m" + limit: + memory: "{{ awx_task_cpu_request|default('2') }}Gi" + cpu: "{{ awx_task_mem_request|default('1500') }}m" - name: awx-rabbit image: ansible/awx_rabbitmq:{{ rabbitmq_version }} imagePullPolicy: Always @@ -81,8 +92,16 @@ spec: value: "awx" - name: RABBITMQ_CONFIG_FILE value: "/etc/rabbitmq/rabbitmq" + resources: + requests: + memory: "2Gi" + cpu: "500m" - name: awx-memcached image: memcached + resources: + requests: + memory: "1Gi" + cpu: "500m" volumes: - name: awx-application-config configMap: