remove memcache from everywhere and add djagno-redis to cover it

CONTRIBUTING.md

@@ -157,7 +157,6 @@ If you start a second terminal session, you can take a look at the running conta
 $ docker ps
 CONTAINER ID        IMAGE                                              COMMAND                  CREATED             STATUS              PORTS                                                                                                                                                              NAMES
 44251b476f98        gcr.io/ansible-tower-engineering/awx_devel:devel   "/entrypoint.sh /bin…"   27 seconds ago      Up 23 seconds       0.0.0.0:6899->6899/tcp, 0.0.0.0:7899-7999->7899-7999/tcp, 0.0.0.0:8013->8013/tcp, 0.0.0.0:8043->8043/tcp, 0.0.0.0:8080->8080/tcp, 22/tcp, 0.0.0.0:8888->8888/tcp   tools_awx_run_9e820694d57e
-b049a43817b4        memcached:alpine                                   "docker-entrypoint.s…"   28 seconds ago      Up 26 seconds       0.0.0.0:11211->11211/tcp                                                                                                                                           tools_memcached_1
 40de380e3c2e        redis:latest                                       "docker-entrypoint.s…"   28 seconds ago      Up 26 seconds       0.0.0.0:6379->6379/tcp                                                                                                                                             tools_redis_1
 b66a506d3007        postgres:10                                        "docker-entrypoint.s…"   28 seconds ago      Up 26 seconds       0.0.0.0:5432->5432/tcp                                                                                                                                             tools_postgres_1
 ```

awx/settings/defaults.py

@@ -441,8 +441,8 @@ CELERYBEAT_SCHEDULE = {
 # Django Caching Configuration
 CACHES = {
     'default': {
-        'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache',
+        'BACKEND': 'django_redis.cache.RedisCache',
-        'LOCATION': 'unix:/var/run/memcached/memcached.sock'
+        'LOCATION': 'unix:/var/run/redis/redis.sock'
     },
 }
 

docs/clustering.md

@@ -218,7 +218,6 @@ Each Tower instance is made up of several different services working collaborati
 * **Callback Receiver** - Receives job events that result from running Ansible jobs.
 * **Celery** - The worker queue that processes and runs all jobs.
 * **Redis** - this is used as a queue for AWX to process ansible playbook callback events.
-* **Memcached** - A local caching service for the instance it lives on.
 
 Tower is configured in such a way that if any of these services or their components fail, then all services are restarted. If these fail sufficiently (often in a short span of time), then the entire instance will be placed offline in an automated fashion in order to allow remediation without causing unexpected behavior.
 

installer/roles/kubernetes/defaults/main.yml

@@ -27,12 +27,6 @@ kubernetes_redis_image: "redis"
 kubernetes_redis_image_tag: "latest"
 kubernetes_redis_config_mount_path: "/usr/local/etc/redis/redis.conf"
 
-memcached_mem_request: 1
-memcached_cpu_request: 500
-
-kubernetes_memcached_version: "latest"
-kubernetes_memcached_image: "memcached"
-
 openshift_pg_emptydir: false
 openshift_pg_pvc_name: postgresql
 

installer/roles/kubernetes/templates/deployment.yml.j2

@@ -161,9 +161,6 @@ spec:
             - name: {{ kubernetes_deployment_name }}-redis-socket
               mountPath: "/var/run/redis"
 
-            - name: {{ kubernetes_deployment_name }}-memcached-socket
-              mountPath: "/var/run/memcached"
-
           resources:
             requests:
               memory: "{{ web_mem_request }}Gi"
@@ -236,9 +233,6 @@ spec:
 
             - name: {{ kubernetes_deployment_name }}-redis-socket
               mountPath: "/var/run/redis"
-
-            - name: {{ kubernetes_deployment_name }}-memcached-socket
-              mountPath: "/var/run/memcached"
           env:
             - name: SUPERVISOR_WEB_CONFIG_PATH
               value: "/etc/supervisord.conf"
@@ -293,31 +287,6 @@ spec:
 {% endif %}
 {% if redis_cpu_limit is defined %}
               cpu: "{{ redis_cpu_limit }}m"
-{% endif %}
-        - name: {{ kubernetes_deployment_name }}-memcached
-          image: "{{ kubernetes_memcached_image }}:{{ kubernetes_memcached_version }}"
-          imagePullPolicy: Always
-          command:
-            - 'memcached'
-            - '-s'
-            - '/var/run/memcached/memcached.sock'
-            - '-a'
-            - '0666'
-          volumeMounts:
-            - name: {{ kubernetes_deployment_name }}-memcached-socket
-              mountPath: "/var/run/memcached"
-          resources:
-            requests:
-              memory: "{{ memcached_mem_request }}Gi"
-              cpu: "{{ memcached_cpu_request }}m"
-{% if memcached_mem_limit is defined or memcached_cpu_limit is defined %}
-            limits:
-{% endif %}
-{% if memcached_mem_limit is defined %}
-              memory: "{{ memcached_mem_limit }}Gi"
-{% endif %}
-{% if memcached_cpu_limit is defined %}
-              cpu: "{{ memcached_cpu_limit }}m"
 {% endif %}
 {% if tolerations is defined %}
       tolerations:
@@ -424,9 +393,6 @@ spec:
         - name: {{ kubernetes_deployment_name }}-redis-socket
           emptyDir: {}
 
-        - name: {{ kubernetes_deployment_name }}-memcached-socket
-          emptyDir: {}
-
 ---
 apiVersion: v1
 kind: Service

installer/roles/local_docker/defaults/main.yml

@@ -7,7 +7,4 @@ redis_image: "redis"
 postgresql_version: "10"
 postgresql_image: "postgres:{{postgresql_version}}"
 
-memcached_image: "memcached"
-memcached_version: "alpine"
-
 compose_start_containers: true

installer/roles/local_docker/tasks/compose.yml

@@ -10,12 +10,6 @@
     state: directory
     mode: 0777
 
-- name: Create Memcached socket directory
-  file:
-    path: "{{ docker_compose_dir }}/memcached_socket"
-    state: directory
-    mode: 0777
-
 - name: Create Docker Compose Configuration
   template:
     src: "{{ item }}.j2"

installer/roles/local_docker/templates/docker-compose.yml.j2

@@ -7,7 +7,7 @@ services:
     container_name: awx_web
     depends_on:
       - redis
-      - memcached
+      - django-redis
       {% if pg_hostname is not defined %}
       - postgres
       {% endif %}
@@ -32,7 +32,6 @@ services:
       - "{{ docker_compose_dir }}/credentials.py:/etc/tower/conf.d/credentials.py"
       - "{{ docker_compose_dir }}/nginx.conf:/etc/nginx/nginx.conf:ro"
       - "{{ docker_compose_dir }}/redis_socket:/var/run/redis/:rw"
-      - "{{ docker_compose_dir }}/memcached_socket:/var/run/memcached/:rw"
     {% if project_data_dir is defined %}
       - "{{ project_data_dir +':/var/lib/awx/projects:rw' }}"
     {% endif %}
@@ -76,7 +75,6 @@ services:
     container_name: awx_task
     depends_on:
       - redis
-      - memcached
       - web
       {% if pg_hostname is not defined %}
       - postgres
@@ -93,7 +91,6 @@ services:
       - "{{ docker_compose_dir }}/environment.sh:/etc/tower/conf.d/environment.sh"
       - "{{ docker_compose_dir }}/credentials.py:/etc/tower/conf.d/credentials.py"
       - "{{ docker_compose_dir }}/redis_socket:/var/run/redis/:rw"
-      - "{{ docker_compose_dir }}/memcached_socket:/var/run/memcached/:rw"
     {% if project_data_dir is defined %}
       - "{{ project_data_dir +':/var/lib/awx/projects:rw' }}"
     {% endif %}
@@ -142,19 +139,6 @@ services:
     volumes:
       - "{{ docker_compose_dir }}/redis.conf:/usr/local/etc/redis/redis.conf:ro"
       - "{{ docker_compose_dir }}/redis_socket:/var/run/redis/:rw"
-      - "{{ docker_compose_dir }}/memcached_socket:/var/run/memcached/:rw"
-
-  memcached:
-    image: "{{ memcached_image }}:{{ memcached_version }}"
-    container_name: awx_memcached
-    command: ["-s", "/var/run/memcached/memcached.sock", "-a", "0666"]
-    restart: unless-stopped
-    environment:
-      http_proxy: {{ http_proxy | default('') }}
-      https_proxy: {{ https_proxy | default('') }}
-      no_proxy: {{ no_proxy | default('') }}
-    volumes:
-      - "{{ docker_compose_dir }}/memcached_socket:/var/run/memcached/:rw"
 
   {% if pg_hostname is not defined %}
   postgres:

requirements/requirements.in

@@ -17,6 +17,7 @@ django-polymorphic
 django-pglocks
 django-qsstats-magic
 django-radius==1.3.3  # FIX auth does not work with later versions
+django-redis
 django-solo
 django-split-settings
 django-taggit
@@ -33,7 +34,6 @@ prometheus_client
 psycopg2
 pygerduty
 pyparsing
-python-memcached
 python-radius
 python3-saml
 pyyaml>=5.3.1  # minimum version to pull in new pyyaml for CVE-2017-18342

requirements/requirements.txt

@@ -32,6 +32,7 @@ django-oauth-toolkit==1.1.3  # via -r /awx_devel/requirements/requirements.in
 django-pglocks==1.0.4     # via -r /awx_devel/requirements/requirements.in
 django-polymorphic==2.1.2  # via -r /awx_devel/requirements/requirements.in
 django-qsstats-magic==1.1.0  # via -r /awx_devel/requirements/requirements.in
+django-redis==4.5.0
 django-radius==1.3.3      # via -r /awx_devel/requirements/requirements.in
 django-solo==1.1.3        # via -r /awx_devel/requirements/requirements.in
 django-split-settings==1.0.0  # via -r /awx_devel/requirements/requirements.in
@@ -93,7 +94,6 @@ pyrsistent==0.15.7        # via jsonschema
 python-daemon==2.2.4      # via ansible-runner
 python-dateutil==2.8.1    # via adal, kubernetes
 python-ldap==3.2.0        # via django-auth-ldap
-python-memcached==1.59    # via -r /awx_devel/requirements/requirements.in
 python-radius==1.0        # via -r /awx_devel/requirements/requirements.in
 python-string-utils==1.0.0  # via openshift
 python3-openid==3.1.0     # via social-auth-core

tools/docker-compose-cluster.yml

@@ -30,7 +30,6 @@ services:
     volumes:
       - "../:/awx_devel"
       - "./redis/redis_socket_ha_1:/var/run/redis/"
-      - "./memcached/:/var/run/memcached"
       - "./docker-compose/supervisor.conf:/etc/supervisord.conf"
     ports:
       - "5899-5999:5899-5999"
@@ -50,7 +49,6 @@ services:
     volumes:
       - "../:/awx_devel"
       - "./redis/redis_socket_ha_2:/var/run/redis/"
-      - "./memcached/:/var/run/memcached"
       - "./docker-compose/supervisor.conf:/etc/supervisord.conf"
     ports:
       - "7899-7999:7899-7999"
@@ -70,7 +68,6 @@ services:
     volumes:
       - "../:/awx_devel"
       - "./redis/redis_socket_ha_3:/var/run/redis/"
-      - "./memcached/:/var/run/memcached"
       - "./docker-compose/supervisor.conf:/etc/supervisord.conf"
     ports:
       - "8899-8999:8899-8999"
@@ -107,10 +104,4 @@ services:
   postgres:
     image: postgres:10
     container_name: tools_postgres_1
-  memcached:
+
-    user: ${CURRENT_UID}
-    image: memcached:alpine
-    container_name: tools_memcached_1
-    command: ["memcached", "-s", "/var/run/memcached/memcached.sock", "-a", "0666"]
-    volumes:
-      - "./memcached/:/var/run/memcached"

tools/docker-compose.yml

@@ -23,7 +23,6 @@ services:
       - "7899-7999:7899-7999"  # default port range for sdb-listen
     links:
       - postgres
-      - memcached
       - redis
     #   - sync
     # volumes_from:
@@ -33,7 +32,7 @@ services:
       - "../:/awx_devel"
       - "../awx/projects/:/var/lib/awx/projects/"
       - "./redis/redis_socket_standalone:/var/run/redis/"
-      - "./memcached/:/var/run/memcached"
+      - "./rsyslog/:/var/lib/awx/rsyslog"
       - "./docker-compose/supervisor.conf:/etc/supervisord.conf"
     privileged: true
     tty: true
@@ -54,13 +53,6 @@ services:
       POSTGRES_HOST_AUTH_METHOD: trust
     volumes:
       - "awx_db:/var/lib/postgresql/data"
-  memcached:
-    user: ${CURRENT_UID}
-    image: memcached:alpine
-    container_name: tools_memcached_1
-    command: ["memcached", "-s", "/var/run/memcached/memcached.sock", "-a", "0666"]
-    volumes:
-      - "./memcached/:/var/run/memcached"
   redis:
     image: redis:latest
     container_name: tools_redis_1