wrap instance and queue registration in postgres advisory locks

see: #7040

awx/main/management/commands/deprovision_node.py

@@ -1,10 +1,14 @@
 # Copyright (c) 2016 Ansible, Inc.
 # All Rights Reserved
 
-import subprocess
-from django.core.management.base import BaseCommand, CommandError
 from optparse import make_option
+import subprocess
+
+from django.db import transaction
+from django.core.management.base import BaseCommand, CommandError
+
 from awx.main.models import Instance
+from awx.main.utils.pglock import advisory_lock
 
 
 class Command(BaseCommand):
@@ -17,19 +21,22 @@ class Command(BaseCommand):
                     help='Hostname used during provisioning'),
     )
 
+    @transaction.atomic
     def handle(self, *args, **options):
-        if not options.get('name'):
+        hostname = options.get('name')
+        if not hostname:
             raise CommandError("--name is a required argument")
-        instance = Instance.objects.filter(hostname=options.get('name'))
+        with advisory_lock('instance_registration_%s' % hostname):
+            instance = Instance.objects.filter(hostname=hostname)
             if instance.exists():
                 instance.delete()
                 print("Instance Removed")
-            result = subprocess.Popen("rabbitmqctl forget_cluster_node rabbitmq@{}".format(options.get('name')), shell=True).wait()
+                result = subprocess.Popen("rabbitmqctl forget_cluster_node rabbitmq@{}".format(hostname), shell=True).wait()
                 if result != 0:
                     print("Node deprovisioning may have failed when attempting to remove the RabbitMQ instance from the cluster")
                 else:
-                print('Successfully deprovisioned {}'.format(options.get('name')))
+                    print('Successfully deprovisioned {}'.format(hostname))
                 print('(changed: True)')
             else:
-            print('No instance found matching name {}'.format(options.get('name')))
+                print('No instance found matching name {}'.format(hostname))
 

awx/main/management/commands/register_instance.py

@@ -2,9 +2,11 @@
 # All Rights Reserved
 
 from awx.main.models import Instance
+from awx.main.utils.pglock import advisory_lock
 from django.conf import settings
 
 from optparse import make_option
+from django.db import transaction
 from django.core.management.base import BaseCommand
 
 
@@ -25,6 +27,7 @@ class Command(BaseCommand):
     def _register_hostname(self, hostname):
         if not hostname:
             return
+        with advisory_lock('instance_registration_%s' % hostname):
             instance = Instance.objects.filter(hostname=hostname)
             if instance.exists():
                 print("Instance already registered {}".format(instance[0]))
@@ -34,6 +37,7 @@ class Command(BaseCommand):
         print('Successfully registered instance {}'.format(hostname))
         self.changed = True
 
+    @transaction.atomic
     def handle(self, **options):
         self.uuid = settings.SYSTEM_UUID
         self.changed = False

awx/main/management/commands/register_queue.py

@@ -2,9 +2,11 @@
 # All Rights Reserved.
 import sys
 
+from awx.main.utils.pglock import advisory_lock
 from awx.main.models import Instance, InstanceGroup
 
 from optparse import make_option
+from django.db import transaction
 from django.core.management.base import BaseCommand, CommandError
 
 
@@ -19,11 +21,14 @@ class Command(BaseCommand):
                     help='The controlling group (makes this an isolated group)'),
     )
 
+    @transaction.atomic
     def handle(self, **options):
-        if not options.get('queuename'):
+        queuename = options.get('queuename')
+        if not queuename:
             raise CommandError("Specify `--queuename` to use this command.")
         changed = False
-        ig = InstanceGroup.objects.filter(name=options.get('queuename'))
+        with advisory_lock('instance_group_registration_%s' % queuename):
+            ig = InstanceGroup.objects.filter(name=queuename)
             control_ig = None
             if options.get('controller'):
                 control_ig = InstanceGroup.objects.filter(name=options.get('controller')).first()
@@ -36,8 +41,8 @@ class Command(BaseCommand):
                     print("Set controller group {} on {}.".format(control_ig.name, ig.name))
                     changed = True
             else:
-            print("Creating instance group {}".format(options.get('queuename')))
+                print("Creating instance group {}".format(queuename))
-            ig = InstanceGroup(name=options.get('queuename'))
+                ig = InstanceGroup(name=queuename)
                 if control_ig:
                     ig.controller = control_ig
                 ig.save()

awx/main/management/commands/unregister_queue.py

@@ -2,9 +2,11 @@
 # All Rights Reserved.
 import sys
 
+from awx.main.utils.pglock import advisory_lock
 from awx.main.models import InstanceGroup
 
 from optparse import make_option
+from django.db import transaction
 from django.core.management.base import BaseCommand, CommandError
 
 
@@ -20,10 +22,13 @@ class Command(BaseCommand):
                     help='Queue to create/update'),
     )
 
+    @transaction.atomic
     def handle(self, **options):
-        if not options.get('queuename'):
+        queuename = options.get('queuename')
+        if not queuename:
             raise CommandError('Must specify `--queuename` in order to use command.')
-        ig = InstanceGroup.objects.filter(name=options.get('queuename'))
+        with advisory_lock('instance_group_registration_%s' % queuename):
+            ig = InstanceGroup.objects.filter(name=queuename)
             if not ig.exists():
                 print("Instance group doesn't exist")
                 sys.exit(1)