From d2bbe7aa1acd6e97138ec736bcde5cfadcf151ae Mon Sep 17 00:00:00 2001 From: Rebeccah Date: Wed, 3 Jun 2020 14:18:19 -0400 Subject: [PATCH] remove memcache from everywhere and add djagno-redis to cover it --- CONTRIBUTING.md | 1 - awx/settings/defaults.py | 4 +-- docs/clustering.md | 1 - installer/roles/kubernetes/defaults/main.yml | 6 ---- .../kubernetes/templates/deployment.yml.j2 | 34 ------------------- .../roles/local_docker/defaults/main.yml | 3 -- .../roles/local_docker/tasks/compose.yml | 6 ---- .../templates/docker-compose.yml.j2 | 18 +--------- requirements/requirements.in | 2 +- requirements/requirements.txt | 2 +- tools/docker-compose-cluster.yml | 11 +----- tools/docker-compose.yml | 10 +----- 12 files changed, 7 insertions(+), 91 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a1aeffb1ba..dc15b0f0f1 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -157,7 +157,6 @@ If you start a second terminal session, you can take a look at the running conta $ docker ps CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES 44251b476f98 gcr.io/ansible-tower-engineering/awx_devel:devel "/entrypoint.sh /bin…" 27 seconds ago Up 23 seconds 0.0.0.0:6899->6899/tcp, 0.0.0.0:7899-7999->7899-7999/tcp, 0.0.0.0:8013->8013/tcp, 0.0.0.0:8043->8043/tcp, 0.0.0.0:8080->8080/tcp, 22/tcp, 0.0.0.0:8888->8888/tcp tools_awx_run_9e820694d57e -b049a43817b4 memcached:alpine "docker-entrypoint.s…" 28 seconds ago Up 26 seconds 0.0.0.0:11211->11211/tcp tools_memcached_1 40de380e3c2e redis:latest "docker-entrypoint.s…" 28 seconds ago Up 26 seconds 0.0.0.0:6379->6379/tcp tools_redis_1 b66a506d3007 postgres:10 "docker-entrypoint.s…" 28 seconds ago Up 26 seconds 0.0.0.0:5432->5432/tcp tools_postgres_1 ``` diff --git a/awx/settings/defaults.py b/awx/settings/defaults.py index b300579471..b737a1d0c4 100644 --- a/awx/settings/defaults.py +++ b/awx/settings/defaults.py @@ -441,8 +441,8 @@ CELERYBEAT_SCHEDULE = { # Django Caching Configuration CACHES = { 'default': { - 'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache', - 'LOCATION': 'unix:/var/run/memcached/memcached.sock' + 'BACKEND': 'django_redis.cache.RedisCache', + 'LOCATION': 'unix:/var/run/redis/redis.sock' }, } diff --git a/docs/clustering.md b/docs/clustering.md index 7b0221beea..ecf9a97dee 100644 --- a/docs/clustering.md +++ b/docs/clustering.md @@ -218,7 +218,6 @@ Each Tower instance is made up of several different services working collaborati * **Callback Receiver** - Receives job events that result from running Ansible jobs. * **Celery** - The worker queue that processes and runs all jobs. * **Redis** - this is used as a queue for AWX to process ansible playbook callback events. -* **Memcached** - A local caching service for the instance it lives on. Tower is configured in such a way that if any of these services or their components fail, then all services are restarted. If these fail sufficiently (often in a short span of time), then the entire instance will be placed offline in an automated fashion in order to allow remediation without causing unexpected behavior. diff --git a/installer/roles/kubernetes/defaults/main.yml b/installer/roles/kubernetes/defaults/main.yml index d300bd7829..5a15a8929a 100644 --- a/installer/roles/kubernetes/defaults/main.yml +++ b/installer/roles/kubernetes/defaults/main.yml @@ -27,12 +27,6 @@ kubernetes_redis_image: "redis" kubernetes_redis_image_tag: "latest" kubernetes_redis_config_mount_path: "/usr/local/etc/redis/redis.conf" -memcached_mem_request: 1 -memcached_cpu_request: 500 - -kubernetes_memcached_version: "latest" -kubernetes_memcached_image: "memcached" - openshift_pg_emptydir: false openshift_pg_pvc_name: postgresql diff --git a/installer/roles/kubernetes/templates/deployment.yml.j2 b/installer/roles/kubernetes/templates/deployment.yml.j2 index c915f5c77f..22ce12153a 100644 --- a/installer/roles/kubernetes/templates/deployment.yml.j2 +++ b/installer/roles/kubernetes/templates/deployment.yml.j2 @@ -161,9 +161,6 @@ spec: - name: {{ kubernetes_deployment_name }}-redis-socket mountPath: "/var/run/redis" - - name: {{ kubernetes_deployment_name }}-memcached-socket - mountPath: "/var/run/memcached" - resources: requests: memory: "{{ web_mem_request }}Gi" @@ -236,9 +233,6 @@ spec: - name: {{ kubernetes_deployment_name }}-redis-socket mountPath: "/var/run/redis" - - - name: {{ kubernetes_deployment_name }}-memcached-socket - mountPath: "/var/run/memcached" env: - name: SUPERVISOR_WEB_CONFIG_PATH value: "/etc/supervisord.conf" @@ -293,31 +287,6 @@ spec: {% endif %} {% if redis_cpu_limit is defined %} cpu: "{{ redis_cpu_limit }}m" -{% endif %} - - name: {{ kubernetes_deployment_name }}-memcached - image: "{{ kubernetes_memcached_image }}:{{ kubernetes_memcached_version }}" - imagePullPolicy: Always - command: - - 'memcached' - - '-s' - - '/var/run/memcached/memcached.sock' - - '-a' - - '0666' - volumeMounts: - - name: {{ kubernetes_deployment_name }}-memcached-socket - mountPath: "/var/run/memcached" - resources: - requests: - memory: "{{ memcached_mem_request }}Gi" - cpu: "{{ memcached_cpu_request }}m" -{% if memcached_mem_limit is defined or memcached_cpu_limit is defined %} - limits: -{% endif %} -{% if memcached_mem_limit is defined %} - memory: "{{ memcached_mem_limit }}Gi" -{% endif %} -{% if memcached_cpu_limit is defined %} - cpu: "{{ memcached_cpu_limit }}m" {% endif %} {% if tolerations is defined %} tolerations: @@ -424,9 +393,6 @@ spec: - name: {{ kubernetes_deployment_name }}-redis-socket emptyDir: {} - - name: {{ kubernetes_deployment_name }}-memcached-socket - emptyDir: {} - --- apiVersion: v1 kind: Service diff --git a/installer/roles/local_docker/defaults/main.yml b/installer/roles/local_docker/defaults/main.yml index 490e1e8fcf..f8e1304702 100644 --- a/installer/roles/local_docker/defaults/main.yml +++ b/installer/roles/local_docker/defaults/main.yml @@ -7,7 +7,4 @@ redis_image: "redis" postgresql_version: "10" postgresql_image: "postgres:{{postgresql_version}}" -memcached_image: "memcached" -memcached_version: "alpine" - compose_start_containers: true diff --git a/installer/roles/local_docker/tasks/compose.yml b/installer/roles/local_docker/tasks/compose.yml index 3212732283..120b81cc1a 100644 --- a/installer/roles/local_docker/tasks/compose.yml +++ b/installer/roles/local_docker/tasks/compose.yml @@ -10,12 +10,6 @@ state: directory mode: 0777 -- name: Create Memcached socket directory - file: - path: "{{ docker_compose_dir }}/memcached_socket" - state: directory - mode: 0777 - - name: Create Docker Compose Configuration template: src: "{{ item }}.j2" diff --git a/installer/roles/local_docker/templates/docker-compose.yml.j2 b/installer/roles/local_docker/templates/docker-compose.yml.j2 index a6cf121593..edfc5e6493 100644 --- a/installer/roles/local_docker/templates/docker-compose.yml.j2 +++ b/installer/roles/local_docker/templates/docker-compose.yml.j2 @@ -7,7 +7,7 @@ services: container_name: awx_web depends_on: - redis - - memcached + - django-redis {% if pg_hostname is not defined %} - postgres {% endif %} @@ -32,7 +32,6 @@ services: - "{{ docker_compose_dir }}/credentials.py:/etc/tower/conf.d/credentials.py" - "{{ docker_compose_dir }}/nginx.conf:/etc/nginx/nginx.conf:ro" - "{{ docker_compose_dir }}/redis_socket:/var/run/redis/:rw" - - "{{ docker_compose_dir }}/memcached_socket:/var/run/memcached/:rw" {% if project_data_dir is defined %} - "{{ project_data_dir +':/var/lib/awx/projects:rw' }}" {% endif %} @@ -76,7 +75,6 @@ services: container_name: awx_task depends_on: - redis - - memcached - web {% if pg_hostname is not defined %} - postgres @@ -93,7 +91,6 @@ services: - "{{ docker_compose_dir }}/environment.sh:/etc/tower/conf.d/environment.sh" - "{{ docker_compose_dir }}/credentials.py:/etc/tower/conf.d/credentials.py" - "{{ docker_compose_dir }}/redis_socket:/var/run/redis/:rw" - - "{{ docker_compose_dir }}/memcached_socket:/var/run/memcached/:rw" {% if project_data_dir is defined %} - "{{ project_data_dir +':/var/lib/awx/projects:rw' }}" {% endif %} @@ -142,19 +139,6 @@ services: volumes: - "{{ docker_compose_dir }}/redis.conf:/usr/local/etc/redis/redis.conf:ro" - "{{ docker_compose_dir }}/redis_socket:/var/run/redis/:rw" - - "{{ docker_compose_dir }}/memcached_socket:/var/run/memcached/:rw" - - memcached: - image: "{{ memcached_image }}:{{ memcached_version }}" - container_name: awx_memcached - command: ["-s", "/var/run/memcached/memcached.sock", "-a", "0666"] - restart: unless-stopped - environment: - http_proxy: {{ http_proxy | default('') }} - https_proxy: {{ https_proxy | default('') }} - no_proxy: {{ no_proxy | default('') }} - volumes: - - "{{ docker_compose_dir }}/memcached_socket:/var/run/memcached/:rw" {% if pg_hostname is not defined %} postgres: diff --git a/requirements/requirements.in b/requirements/requirements.in index c847496844..b03e163e3c 100644 --- a/requirements/requirements.in +++ b/requirements/requirements.in @@ -17,6 +17,7 @@ django-polymorphic django-pglocks django-qsstats-magic django-radius==1.3.3 # FIX auth does not work with later versions +django-redis django-solo django-split-settings django-taggit @@ -33,7 +34,6 @@ prometheus_client psycopg2 pygerduty pyparsing -python-memcached python-radius python3-saml pyyaml>=5.3.1 # minimum version to pull in new pyyaml for CVE-2017-18342 diff --git a/requirements/requirements.txt b/requirements/requirements.txt index 00fbee267d..0d364a4922 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -32,6 +32,7 @@ django-oauth-toolkit==1.1.3 # via -r /awx_devel/requirements/requirements.in django-pglocks==1.0.4 # via -r /awx_devel/requirements/requirements.in django-polymorphic==2.1.2 # via -r /awx_devel/requirements/requirements.in django-qsstats-magic==1.1.0 # via -r /awx_devel/requirements/requirements.in +django-redis==4.5.0 django-radius==1.3.3 # via -r /awx_devel/requirements/requirements.in django-solo==1.1.3 # via -r /awx_devel/requirements/requirements.in django-split-settings==1.0.0 # via -r /awx_devel/requirements/requirements.in @@ -93,7 +94,6 @@ pyrsistent==0.15.7 # via jsonschema python-daemon==2.2.4 # via ansible-runner python-dateutil==2.8.1 # via adal, kubernetes python-ldap==3.2.0 # via django-auth-ldap -python-memcached==1.59 # via -r /awx_devel/requirements/requirements.in python-radius==1.0 # via -r /awx_devel/requirements/requirements.in python-string-utils==1.0.0 # via openshift python3-openid==3.1.0 # via social-auth-core diff --git a/tools/docker-compose-cluster.yml b/tools/docker-compose-cluster.yml index 95f7f5aaa8..9b9bac1fdd 100644 --- a/tools/docker-compose-cluster.yml +++ b/tools/docker-compose-cluster.yml @@ -30,7 +30,6 @@ services: volumes: - "../:/awx_devel" - "./redis/redis_socket_ha_1:/var/run/redis/" - - "./memcached/:/var/run/memcached" - "./docker-compose/supervisor.conf:/etc/supervisord.conf" ports: - "5899-5999:5899-5999" @@ -50,7 +49,6 @@ services: volumes: - "../:/awx_devel" - "./redis/redis_socket_ha_2:/var/run/redis/" - - "./memcached/:/var/run/memcached" - "./docker-compose/supervisor.conf:/etc/supervisord.conf" ports: - "7899-7999:7899-7999" @@ -70,7 +68,6 @@ services: volumes: - "../:/awx_devel" - "./redis/redis_socket_ha_3:/var/run/redis/" - - "./memcached/:/var/run/memcached" - "./docker-compose/supervisor.conf:/etc/supervisord.conf" ports: - "8899-8999:8899-8999" @@ -107,10 +104,4 @@ services: postgres: image: postgres:10 container_name: tools_postgres_1 - memcached: - user: ${CURRENT_UID} - image: memcached:alpine - container_name: tools_memcached_1 - command: ["memcached", "-s", "/var/run/memcached/memcached.sock", "-a", "0666"] - volumes: - - "./memcached/:/var/run/memcached" + diff --git a/tools/docker-compose.yml b/tools/docker-compose.yml index b1e8fb93c7..aadc953f6e 100644 --- a/tools/docker-compose.yml +++ b/tools/docker-compose.yml @@ -23,7 +23,6 @@ services: - "7899-7999:7899-7999" # default port range for sdb-listen links: - postgres - - memcached - redis # - sync # volumes_from: @@ -33,7 +32,7 @@ services: - "../:/awx_devel" - "../awx/projects/:/var/lib/awx/projects/" - "./redis/redis_socket_standalone:/var/run/redis/" - - "./memcached/:/var/run/memcached" + - "./rsyslog/:/var/lib/awx/rsyslog" - "./docker-compose/supervisor.conf:/etc/supervisord.conf" privileged: true tty: true @@ -54,13 +53,6 @@ services: POSTGRES_HOST_AUTH_METHOD: trust volumes: - "awx_db:/var/lib/postgresql/data" - memcached: - user: ${CURRENT_UID} - image: memcached:alpine - container_name: tools_memcached_1 - command: ["memcached", "-s", "/var/run/memcached/memcached.sock", "-a", "0666"] - volumes: - - "./memcached/:/var/run/memcached" redis: image: redis:latest container_name: tools_redis_1