Add a periodic celery task to clean up local stdout files

This works in tandem with the on-demand stdout file generator and
decouples stdout locally from the persistent database representation
ob the job itself

awx/main/tasks.py

@@ -59,7 +59,7 @@ from awx.main.consumers import emit_channel_notification
 __all__ = ['RunJob', 'RunSystemJob', 'RunProjectUpdate', 'RunInventoryUpdate',
            'RunAdHocCommand', 'handle_work_error',
            'handle_work_success', 'update_inventory_computed_fields',
-           'send_notifications', 'run_administrative_checks']
+           'send_notifications', 'run_administrative_checks', 'purge_old_stdout_files']
 
 HIDDEN_PASSWORD = '**********'
 
@@ -193,6 +193,15 @@ def cleanup_authtokens(self):
     AuthToken.objects.filter(expires__lt=now()).delete()
 
 
+@task(bind=True)
+def purge_old_stdout_files(self):
+    nowtime = time.time()
+    for f in os.listdir(settings.JOBOUTPUT_ROOT):
+        if os.path.getctime(os.path.join(settings.JOBOUTPUT_ROOT,f)) < nowtime - settings.LOCAL_STDOUT_EXPIRE_TIME:
+            os.unlink(os.path.join(settings.JOBOUTPUT_ROOT,f))
+            logger.info("Removing {}".format(os.path.join(settings.JOBOUTPUT_ROOT,f)))
+
+
 @task(bind=True)
 def cluster_node_heartbeat(self):
     logger.debug("Cluster node heartbeat task.")

awx/settings/defaults.py

@@ -163,8 +163,15 @@ MAX_EVENT_RES_DATA = 700000
 # Note: This setting may be overridden by database settings.
 EVENT_STDOUT_MAX_BYTES_DISPLAY = 1024
 
+# The amount of time before a stdout file is expired and removed locally
+# Note that this can be recreated if the stdout is downloaded
+LOCAL_STDOUT_EXPIRE_TIME = 2592000
+
+# The number of processes spawned by the callback receiver to process job
+# events into the database
 JOB_EVENT_WORKERS = 4
 
+# The maximum size of the job event worker queue before requests are blocked
 JOB_EVENT_MAX_QUEUE_SIZE = 10000
 
 # Disallow sending session cookies over insecure connections
@@ -416,6 +423,8 @@ CELERY_ROUTES = {'awx.main.tasks.run_job': {'queue': 'jobs',
                  'awx.main.scheduler.tasks.run_job_complete': {'queue': 'scheduler',
                                                                'routing_key': 'scheduler.job.complete'},
                  'awx.main.tasks.cluster_node_heartbeat': {'queue': 'default',
+                                                           'routing_key': 'cluster.heartbeat'},
+                 'awx.main.tasks.purge_old_stdout_files': {'queue': 'default',
                                                            'routing_key': 'cluster.heartbeat'}}
 
 CELERYBEAT_SCHEDULE = {
@@ -435,6 +444,10 @@ CELERYBEAT_SCHEDULE = {
         'task': 'awx.main.tasks.cluster_node_heartbeat',
         'schedule': timedelta(seconds=60)
     },
+    'purge_stdout_files': {
+        'task': 'awx.main.tasks.purge_old_stdout_files',
+        'schedule': timedelta(days=7)
+    },
     'task_manager': {
         'task': 'awx.main.scheduler.tasks.run_task_manager',
         'schedule': timedelta(seconds=20)