Merge pull request #90 from matburt/scan_jobs_in_tower

Scan jobs in tower

awx/api/serializers.py

@@ -745,6 +745,7 @@ class InventorySerializer(BaseSerializerWithVariables):
             tree          = reverse('api:inventory_tree_view',         args=(obj.pk,)),
             inventory_sources = reverse('api:inventory_inventory_sources_list', args=(obj.pk,)),
             activity_stream = reverse('api:inventory_activity_stream_list', args=(obj.pk,)),
+            scan_job_templates = reverse('api:inventory_scan_job_template_list', args=(obj.pk,)),
         ))
         if obj.organization and obj.organization.active:
             res['organization'] = reverse('api:organization_detail', args=(obj.organization.pk,))
@@ -1304,11 +1305,19 @@ class JobOptionsSerializer(BaseSerializer):
             ret['cloud_credential'] = None
         return ret
 
+    def validate_project(self, attrs, source):
+        project = attrs.get('project', None)
+        if not project and attrs.get('job_type') != PERM_INVENTORY_SCAN:
+            raise serializers.ValidationError("This field is required")
+        return attrs
+
     def validate_playbook(self, attrs, source):
         project = attrs.get('project', None)
         playbook = attrs.get('playbook', '')
         if project and playbook and smart_str(playbook) not in project.playbooks:
             raise serializers.ValidationError('Playbook not found for project')
+        if project and not playbook:
+            raise serializers.ValidationError('Must select playbook for project')
         return attrs
 
 
@@ -1470,14 +1479,12 @@ class SystemJobSerializer(UnifiedJobSerializer):
                                                  args=(obj.system_job_template.pk,))
         return res
 
-
 class JobListSerializer(JobSerializer, UnifiedJobListSerializer):
     pass
 
 class SystemJobListSerializer(SystemJobSerializer, UnifiedJobListSerializer):
     pass
 
-
 class JobHostSummarySerializer(BaseSerializer):
 
     class Meta:

awx/api/urls.py

@@ -73,6 +73,7 @@ inventory_urls = patterns('awx.api.views',
     url(r'^(?P<pk>[0-9]+)/tree/$',                      'inventory_tree_view'),
     url(r'^(?P<pk>[0-9]+)/inventory_sources/$',         'inventory_inventory_sources_list'),
     url(r'^(?P<pk>[0-9]+)/activity_stream/$',           'inventory_activity_stream_list'),
+    url(r'^(?P<pk>[0-9]+)/scan_job_templates/$',        'inventory_scan_job_template_list'),
 )
 
 host_urls = patterns('awx.api.views',

awx/api/views.py

@@ -907,6 +907,20 @@ class InventoryActivityStreamList(SubListAPIView):
         qs = self.request.user.get_queryset(self.model)
         return qs.filter(Q(inventory=parent) | Q(host__in=parent.hosts.all()) | Q(group__in=parent.groups.all()))
 
+class InventoryScanJobTemplateList(SubListAPIView):
+
+    model = JobTemplate
+    serializer_class = JobTemplateSerializer
+    parent_model = Inventory
+    relationship = 'jobtemplates'
+    new_in_220 = True
+
+    def get_queryset(self):
+        parent = self.get_parent_object()
+        self.check_parent_access(parent)
+        qs = self.request.user.get_queryset(self.model)
+        return qs.filter(job_type=PERM_INVENTORY_SCAN, inventory=parent)
+
 
 class HostList(ListCreateAPIView):
 
@@ -1451,7 +1465,7 @@ class JobTemplateLaunch(GenericAPIView):
                                 status=status.HTTP_400_BAD_REQUEST)
         if obj.credential is None and ('credential' not in request.DATA and 'credential_id' not in request.DATA):
             return Response(dict(errors="Credential not provided"), status=status.HTTP_400_BAD_REQUEST)
-        if obj.project is None or not obj.project.active:
+        if obj.job_type != PERM_INVENTORY_SCAN and (obj.project is None or not obj.project.active):
             return Response(dict(errors="Job Template Project is missing or undefined"), status=status.HTTP_400_BAD_REQUEST)
         if obj.inventory is None or not obj.inventory.active:
             return Response(dict(errors="Job Template Inventory is missing or undefined"), status=status.HTTP_400_BAD_REQUEST)

awx/main/access.py

@@ -824,7 +824,7 @@ class PermissionAccess(BaseAccess):
         return self.can_change(obj, None)
 
 class JobTemplateAccess(BaseAccess):
-    ''' 
+    '''
     I can see job templates when:
      - I am a superuser.
      - I can read the inventory, project and credential (which means I am an
@@ -851,8 +851,10 @@ class JobTemplateAccess(BaseAccess):
         )
         # FIXME: Check active status on related objects!
         org_admin_qs = base_qs.filter(
-            project__organizations__admins__in=[self.user]
+            Q(project__organizations__admins__in=[self.user]) |
+            (Q(project__isnull=True) & Q(job_type=PERM_INVENTORY_SCAN) & Q(inventory__organization__admins__in=[self.user]))
         )
+
         allowed_deploy = [PERM_JOBTEMPLATE_CREATE, PERM_INVENTORY_DEPLOY]
         allowed_check = [PERM_JOBTEMPLATE_CREATE, PERM_INVENTORY_DEPLOY, PERM_INVENTORY_CHECK]
 
@@ -923,10 +925,15 @@ class JobTemplateAccess(BaseAccess):
         inventory = Inventory.objects.filter(id=inventory_pk)
         if not inventory.exists():
             return False # Does this make sense?  Maybe should check read access
-        
+
+        project_pk = get_pk_from_dict(data, 'project')
+        if 'job_type' in data and data['job_type'] == PERM_INVENTORY_SCAN:
+            if not project_pk and self.user.can_access(Organization, 'change', inventory[0].organization, None):
+                return True
+            elif not self.user.can_access(Organization, "change", inventory[0].organization, None):
+                return False
         # If the user has admin access to the project (as an org admin), should
         # be able to proceed without additional checks.
-        project_pk = get_pk_from_dict(data, 'project')
         project = get_object_or_400(Project, pk=project_pk)
         if self.user.can_access(Project, 'admin', project, None):
             return True
@@ -944,9 +951,9 @@ class JobTemplateAccess(BaseAccess):
         if permission_qs.exists():
             return True
         return False
-            
+
         # job_type = data.get('job_type', None)
-        
+
         # for perm in permission_qs:
         #     # if you have run permissions, you can also create check jobs
         #     if job_type == PERM_INVENTORY_CHECK:
@@ -985,7 +992,14 @@ class JobTemplateAccess(BaseAccess):
         if self.user.is_superuser:
             return True
         # Check to make sure both the inventory and project exist
-        if obj.inventory is None or obj.project is None:
+        if obj.inventory is None:
+            return False
+        if obj.job_type == PERM_INVENTORY_SCAN:
+            if obj.project is None and self.user.can_access(Organization, 'change', obj.inventory.organization, None):
+                return True
+            if not self.user.can_access(Organization, 'change', obj.inventory.organization, None):
+                return False
+        if obj.project is None:
             return False
         # If the user has admin access to the project they can start a job
         if self.user.can_access(Project, 'admin', obj.project, None):
@@ -1011,7 +1025,7 @@ class JobTemplateAccess(BaseAccess):
             if perm.permission_type in [PERM_JOBTEMPLATE_CREATE, PERM_INVENTORY_DEPLOY, PERM_INVENTORY_CHECK] and \
                obj.job_type == PERM_INVENTORY_CHECK:
                 has_perm = True
-                
+
         dep_access = self.user.can_access(Inventory, 'read', obj.inventory) and self.user.can_access(Project, 'read', obj.project)
         return dep_access and has_perm
 
@@ -1029,7 +1043,8 @@ class JobTemplateAccess(BaseAccess):
         add_obj = dict(credential=obj.credential.id if obj.credential is not None else None,
                        cloud_credential=obj.cloud_credential.id if obj.cloud_credential is not None else None,
                        inventory=obj.inventory.id if obj.inventory is not None else None,
-                       project=obj.project.id if obj.project is not None else None)
+                       project=obj.project.id if obj.project is not None else None,
+                       job_type=obj.job_type)
         return self.can_add(add_obj)
 
 class JobAccess(BaseAccess):
@@ -1048,8 +1063,10 @@ class JobAccess(BaseAccess):
             credential_id__in=credential_ids,
         )
         org_admin_qs = base_qs.filter(
-            project__organizations__admins__in=[self.user]
+            Q(project__organizations__admins__in=[self.user]) |
+            (Q(project__isnull=True) & Q(job_type=PERM_INVENTORY_SCAN) & Q(inventory__organization__admins__in=[self.user]))
         )
+
         allowed_deploy = [PERM_JOBTEMPLATE_CREATE, PERM_INVENTORY_DEPLOY]
         allowed_check = [PERM_JOBTEMPLATE_CREATE, PERM_INVENTORY_DEPLOY, PERM_INVENTORY_CHECK]
         team_ids = set(Team.objects.filter(users__in=[self.user]).values_list('id', flat=True))
@@ -1148,8 +1165,9 @@ class JobAccess(BaseAccess):
         has_perm = False
         if obj.job_template is not None and self.user.can_access(JobTemplate, 'start', obj.job_template):
             has_perm = True
-        dep_access = self.user.can_access(Inventory, 'read', obj.inventory) and self.user.can_access(Project, 'read', obj.project)
+        dep_access_inventory = self.user.can_access(Inventory, 'read', obj.inventory)
-        return self.can_read(obj) and dep_access and has_perm
+        dep_access_project = obj.project is None or self.user.can_access(Project, 'read', obj.project)
+        return self.can_read(obj) and dep_access_inventory and dep_access_project and has_perm
 
     def can_cancel(self, obj):
         return self.can_read(obj) and obj.can_cancel

awx/main/models/base.py

@@ -27,7 +27,7 @@ __all__ = ['VarsDictProperty', 'BaseModel', 'CreatedModifiedModel',
            'PasswordFieldsModel', 'PrimordialModel', 'CommonModel',
            'CommonModelNameNotUnique',
            'PERM_INVENTORY_ADMIN', 'PERM_INVENTORY_READ',
-           'PERM_INVENTORY_WRITE', 'PERM_INVENTORY_DEPLOY',
+           'PERM_INVENTORY_WRITE', 'PERM_INVENTORY_DEPLOY', 'PERM_INVENTORY_SCAN',
            'PERM_INVENTORY_CHECK', 'PERM_JOBTEMPLATE_CREATE', 'JOB_TYPE_CHOICES',
            'PERMISSION_TYPE_CHOICES', 'CLOUD_INVENTORY_SOURCES']
 
@@ -36,11 +36,13 @@ PERM_INVENTORY_READ   = 'read'
 PERM_INVENTORY_WRITE  = 'write'
 PERM_INVENTORY_DEPLOY = 'run'
 PERM_INVENTORY_CHECK  = 'check'
+PERM_INVENTORY_SCAN   = 'scan'
 PERM_JOBTEMPLATE_CREATE = 'create'
 
 JOB_TYPE_CHOICES = [
     (PERM_INVENTORY_DEPLOY, _('Run')),
     (PERM_INVENTORY_CHECK, _('Check')),
+    (PERM_INVENTORY_SCAN, _('Scan')),
 ]
 
 PERMISSION_TYPE_CHOICES = [
@@ -49,6 +51,7 @@ PERMISSION_TYPE_CHOICES = [
     (PERM_INVENTORY_ADMIN, _('Administrate Inventory')),
     (PERM_INVENTORY_DEPLOY, _('Deploy To Inventory')),
     (PERM_INVENTORY_CHECK, _('Deploy To Inventory (Dry Run)')),
+    (PERM_INVENTORY_SCAN, _('Scan an Inventory')),
     (PERM_JOBTEMPLATE_CREATE, _('Create a Job Template')),
 ]
 

awx/main/models/jobs.py

@@ -52,11 +52,14 @@ class JobOptions(BaseModel):
         'Project',
         related_name='%(class)ss',
         null=True,
+        default=None,
+        blank=True,
         on_delete=models.SET_NULL,
     )
     playbook = models.CharField(
         max_length=1024,
         default='',
+        blank=True,
     )
     credential = models.ForeignKey(
         'Credential',
@@ -142,7 +145,6 @@ class JobOptions(BaseModel):
                     needed.append(pw)
         return needed
 
-
 class JobTemplate(UnifiedJobTemplate, JobOptions):
     '''
     A job template is a reusable job definition for applying a project (with
@@ -1016,7 +1018,6 @@ class SystemJob(UnifiedJob, SystemJobOptions):
     def is_blocked_by(self, obj):
         return True
 
-    
     def handle_extra_data(self, extra_data):
         extra_vars = {}
         if type(extra_data) == dict:
@@ -1037,3 +1038,4 @@ class SystemJob(UnifiedJob, SystemJobOptions):
     @property
     def task_impact(self):
         return 150
+

awx/main/tasks.py

@@ -550,10 +550,6 @@ class RunJob(BaseTask):
         env['JOB_ID'] = str(job.pk)
         env['INVENTORY_ID'] = str(job.inventory.pk)
         env['ANSIBLE_CALLBACK_PLUGINS'] = plugin_dir
-        # TODO: env['ANSIBLE_LIBRARY'] # plugins/library
-        # TODO: env['ANSIBLE_CACHE_PLUGINS'] # plugins/fact_caching
-        # TODD: env['ANSIBLE_CACHE_PLUGIN'] # tower
-        # TODO: env['ANSIBLE_CACHE_PLUGIN_CONNECTION'] # connection to tower service
         env['REST_API_URL'] = settings.INTERNAL_API_URL
         env['REST_API_TOKEN'] = job.task_auth_token or ''
         env['CALLBACK_CONSUMER_PORT'] = str(settings.CALLBACK_CONSUMER_PORT)
@@ -598,6 +594,12 @@ class RunJob(BaseTask):
             env['VMWARE_PASSWORD'] = decrypt_field(cloud_cred, 'password')
             env['VMWARE_HOST'] = cloud_cred.host
 
+        # Set environment variables related to scan jobs
+        if job.job_type == PERM_INVENTORY_SCAN:
+            env['ANSIBLE_LIBRARY'] = self.get_path_to('..', 'plugins', 'library')
+            env['ANSIBLE_CACHE_PLUGINS'] = self.get_path_to('..', 'plugins', 'fact_caching')
+            env['ANSIBLE_CACHE_PLUGIN'] = "tower"
+            env['ANSIBLE_CACHE_PLUGIN_CONNECTION'] = "tcp://127.0.0.1:%s" % str(settings.FACT_CACHE_PORT)
         return env
 
     def build_args(self, job, **kwargs):
@@ -678,11 +680,16 @@ class RunJob(BaseTask):
         args.extend(['-e', json.dumps(extra_vars)])
 
         # Add path to playbook (relative to project.local_path).
-        args.append(job.playbook)
+        if job.project is None and job.job_type == PERM_INVENTORY_SCAN:
+            args.append("scan_facts.yml")
+        else:
+            args.append(job.playbook)
 
         return args
 
     def build_cwd(self, job, **kwargs):
+        if job.project is None and job.job_type == PERM_INVENTORY_SCAN:
+            return self.get_path_to('..', 'playbooks')
         cwd = job.project.get_project_path()
         if not cwd:
             root = settings.PROJECTS_ROOT

awx/main/tests/jobs/jobs_monolithic.py

@@ -423,6 +423,27 @@ class JobTemplateTest(BaseJobTestMixin, django.test.TestCase):
 
         # FIXME: Check other credentials and optional fields.
 
+    def test_post_scan_job_template(self):
+        url = reverse('api:job_template_list')
+        data = dict(
+            name = 'scan job template 1',
+            job_type = PERM_INVENTORY_SCAN,
+            inventory = self.inv_eng.pk,
+        )
+        # Regular users, even those who have access to the inv and cred can't create scan jobs templates
+        with self.current_user(self.user_doug):
+            data['credential'] = self.cred_doug.pk
+            response = self.post(url, data, expect=403)
+        # Org admins can create scan job templates in their org
+        with self.current_user(self.user_chuck):
+            data['credential'] = self.cred_chuck.pk
+            response = self.post(url, data, expect=201)
+            detail_url = reverse('api:job_template_detail',
+                                 args=(response['id'],))
+        # Non Org Admins don't have permission to access it though
+        with self.current_user(self.user_doug):
+            self.get(detail_url, expect=403)
+
     def test_launch_job_template(self):
         url = reverse('api:job_template_list')
         data = dict(

awx/plugins/fact_caching/tower.py

@@ -45,6 +45,9 @@ class CacheModule(BaseCacheModule):
 
     def __init__(self, *args, **kwargs):
 
+        # Basic in-memory caching for typical runs
+        self._cache = {}
+
         # This is the local tower zmq connection
         self._tower_connection = C.CACHE_PLUGIN_CONNECTION
         self.date_key = time.mktime(datetime.datetime.utcnow().timetuple())
@@ -58,23 +61,26 @@ class CacheModule(BaseCacheModule):
             sys.exit(1)
 
     def get(self, key):
-        return {} # Temporary until we have some tower retrieval endpoints
+        return self._cache.get(key)
 
     def set(self, key, value):
+        self._cache[key] = value
+
+        # Emit fact data to tower for processing
         self.socket.send_json(dict(host=key, facts=value, date_key=self.date_key))
         self.socket.recv()
 
     def keys(self):
-        return []
+        return self._cache.keys()
 
     def contains(self, key):
-        return False
+        return key in self._cache
 
     def delete(self, key):
-        pass
+        del self._cache[key]
 
     def flush(self):
-        pass
+        self._cache = {}
 
     def copy(self):
-        return dict()
+        return self._cache.copy()

awx/plugins/library/scan_packages.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 import os
-from ansible.module_utils.basic import *
+from ansible.module_utils.basic import * # noqa
 
 def rpm_package_list():
     import rpm