Merge pull request #7911 from AmadeusITGroup/archive_url_scm_type

Add Remote Archive SCM Type to support using artifacts and releases as projects

Reviewed-by: https://github.com/apps/softwarefactory-project-zuul

CHANGELOG.md

@@ -4,6 +4,7 @@ This is a list of high-level changes for each release of AWX. A full list of com
 
 ## 14.1.0 (TBD)
 - AWX images can now be built on ARM64 - https://github.com/ansible/awx/pull/7607
+- Added the Remote Archive SCM Type to support using immutable artifacts and releases (such as tarballs and zip files) as projects - https://github.com/ansible/awx/issues/7954
 - Deprecated official support for Mercurial-based project updates - https://github.com/ansible/awx/issues/7932
 - Added resource import/export support to the official AWX collection - https://github.com/ansible/awx/issues/7329
 - Added the ability to import YAML-based resources (instead of just JSON) when using the AWX CLI - https://github.com/ansible/awx/pull/7808

awx/api/serializers.py

@@ -1336,6 +1336,8 @@ class ProjectOptionsSerializer(BaseSerializer):
             attrs.pop('local_path', None)
         if 'local_path' in attrs and attrs['local_path'] not in valid_local_paths:
             errors['local_path'] = _('This path is already being used by another manual project.')
+        if attrs.get('scm_branch') and scm_type == 'archive':
+            errors['scm_branch'] = _('SCM branch cannot be used with archive projects.')
         if attrs.get('scm_refspec') and scm_type != 'git':
             errors['scm_refspec'] = _('SCM refspec can only be used with git projects.')
 

awx/api/views/__init__.py

@@ -242,6 +242,8 @@ class DashboardView(APIView):
         svn_failed_projects = svn_projects.filter(last_job_failed=True)
         hg_projects = user_projects.filter(scm_type='hg')
         hg_failed_projects = hg_projects.filter(last_job_failed=True)
+        archive_projects = user_projects.filter(scm_type='archive')
+        archive_failed_projects = archive_projects.filter(last_job_failed=True)
         data['scm_types'] = {}
         data['scm_types']['git'] = {'url': reverse('api:project_list', request=request) + "?scm_type=git",
                                     'label': 'Git',
@@ -258,6 +260,11 @@ class DashboardView(APIView):
                                    'failures_url': reverse('api:project_list', request=request) + "?scm_type=hg&last_job_failed=True",
                                    'total': hg_projects.count(),
                                    'failed': hg_failed_projects.count()}
+        data['scm_types']['archive'] = {'url': reverse('api:project_list', request=request) + "?scm_type=archive",
+                                        'label': 'Remote Archive',
+                                        'failures_url': reverse('api:project_list', request=request) + "?scm_type=archive&last_job_failed=True",
+                                        'total': archive_projects.count(),
+                                        'failed': archive_failed_projects.count()}
 
         user_list = get_user_queryset(request.user, models.User)
         team_list = get_user_queryset(request.user, models.Team)

awx/main/migrations/0118_add_remote_archive_scm_type.py

@@ -0,0 +1,23 @@
+# Generated by Django 2.2.11 on 2020-08-18 22:35
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('main', '0117_v400_remove_cloudforms_inventory'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='project',
+            name='scm_type',
+            field=models.CharField(blank=True, choices=[('', 'Manual'), ('git', 'Git'), ('hg', 'Mercurial'), ('svn', 'Subversion'), ('insights', 'Red Hat Insights'), ('archive', 'Remote Archive')], default='', help_text='Specifies the source control system used to store the project.', max_length=8, verbose_name='SCM Type'),
+        ),
+        migrations.AlterField(
+            model_name='projectupdate',
+            name='scm_type',
+            field=models.CharField(blank=True, choices=[('', 'Manual'), ('git', 'Git'), ('hg', 'Mercurial'), ('svn', 'Subversion'), ('insights', 'Red Hat Insights'), ('archive', 'Remote Archive')], default='', help_text='Specifies the source control system used to store the project.', max_length=8, verbose_name='SCM Type'),
+        ),
+    ]

awx/main/models/projects.py

@@ -55,6 +55,7 @@ class ProjectOptions(models.Model):
         ('hg', _('Mercurial')),
         ('svn', _('Subversion')),
         ('insights', _('Red Hat Insights')),
+        ('archive', _('Remote Archive')),
     ]
 
     class Meta:

awx/main/tasks.py

@@ -2105,7 +2105,7 @@ class RunProjectUpdate(BaseTask):
                     scm_username = False
             elif scm_url_parts.scheme.endswith('ssh'):
                 scm_password = False
-            elif scm_type == 'insights':
+            elif scm_type in ('insights', 'archive'):
                 extra_vars['scm_username'] = scm_username
                 extra_vars['scm_password'] = scm_password
             scm_url = update_scm_url(scm_type, scm_url, scm_username,

awx/main/tests/functional/analytics/test_projects_by_scm_type.py

@@ -12,7 +12,8 @@ def test_empty():
         'git': 0,
         'svn': 0,
         'hg': 0,
-        'insights': 0
+        'insights': 0,
+        'archive': 0,
     }
 
 
@@ -24,7 +25,8 @@ def test_multiple(scm_type):
         'git': 0,
         'svn': 0,
         'hg': 0,
-        'insights': 0
+        'insights': 0,
+        'archive': 0,
     }
     for i in range(random.randint(0, 10)):
         Project(scm_type=scm_type).save()

awx/main/tests/unit/test_tasks.py

@@ -1792,16 +1792,19 @@ class TestProjectUpdateCredentials(TestJobExecution):
             dict(scm_type='git'),
             dict(scm_type='hg'),
             dict(scm_type='svn'),
+            dict(scm_type='archive'),
         ],
         'test_ssh_key_auth': [
             dict(scm_type='git'),
             dict(scm_type='hg'),
             dict(scm_type='svn'),
+            dict(scm_type='archive'),
         ],
         'test_awx_task_env': [
             dict(scm_type='git'),
             dict(scm_type='hg'),
             dict(scm_type='svn'),
+            dict(scm_type='archive'),
         ]
     }
 

awx/main/utils/common.py

@@ -257,7 +257,7 @@ def update_scm_url(scm_type, url, username=True, password=True,
     # git: https://www.kernel.org/pub/software/scm/git/docs/git-clone.html#URLS
     # hg: http://www.selenic.com/mercurial/hg.1.html#url-paths
     # svn: http://svnbook.red-bean.com/en/1.7/svn-book.html#svn.advanced.reposurls
-    if scm_type not in ('git', 'hg', 'svn', 'insights'):
+    if scm_type not in ('git', 'hg', 'svn', 'insights', 'archive'):
         raise ValueError(_('Unsupported SCM type "%s"') % str(scm_type))
     if not url.strip():
         return ''
@@ -303,7 +303,8 @@ def update_scm_url(scm_type, url, username=True, password=True,
         'git': ('ssh', 'git', 'git+ssh', 'http', 'https', 'ftp', 'ftps', 'file'),
         'hg': ('http', 'https', 'ssh', 'file'),
         'svn': ('http', 'https', 'svn', 'svn+ssh', 'file'),
-        'insights': ('http', 'https')
+        'insights': ('http', 'https'),
+        'archive': ('http', 'https'),
     }
     if parts.scheme not in scm_type_schemes.get(scm_type, ()):
         raise ValueError(_('Unsupported %s URL') % scm_type)
@@ -339,7 +340,7 @@ def update_scm_url(scm_type, url, username=True, password=True,
             #raise ValueError('Password not supported for SSH with Mercurial.')
             netloc_password = ''
 
-    if netloc_username and parts.scheme != 'file' and scm_type != "insights":
+    if netloc_username and parts.scheme != 'file' and scm_type not in ("insights", "archive"):
         netloc = u':'.join([urllib.parse.quote(x,safe='') for x in (netloc_username, netloc_password) if x])
     else:
         netloc = u''

awx/playbooks/action_plugins/project_archive.py

@@ -0,0 +1,82 @@
+from __future__ import absolute_import, division, print_function
+
+__metaclass__ = type
+
+import zipfile
+import tarfile
+import os
+
+from ansible.plugins.action import ActionBase
+from ansible.utils.display import Display
+
+display = Display()
+
+
+class ActionModule(ActionBase):
+    def run(self, tmp=None, task_vars=None):
+        self._supports_check_mode = False
+
+        result = super(ActionModule, self).run(tmp, task_vars)
+
+        src = self._task.args.get("src")
+        proj_path = self._task.args.get("project_path")
+        force = self._task.args.get("force", False)
+
+        try:
+            archive = zipfile.ZipFile(src)
+            get_filenames = archive.namelist
+            get_members = archive.infolist
+        except zipfile.BadZipFile:
+            archive = tarfile.open(src)
+            get_filenames = archive.getnames
+            get_members = archive.getmembers
+        except tarfile.ReadError:
+            result["failed"] = True
+            result["msg"] = "{0} is not a valid archive".format(src)
+            return result
+
+        # Most well formed archives contain a single root directory, typically named
+        # project-name-1.0.0. The project contents should be inside that directory.
+        start_index = 0
+        root_contents = set(
+            [filename.split(os.path.sep)[0] for filename in get_filenames()]
+        )
+        if len(root_contents) == 1:
+            start_index = len(list(root_contents)[0]) + 1
+
+        for member in get_members():
+            try:
+                filename = member.filename
+            except AttributeError:
+                filename = member.name
+
+            # Skip the archive base directory
+            if not filename[start_index:]:
+                continue
+
+            dest = os.path.join(proj_path, filename[start_index:])
+
+            if not force and os.path.exists(dest):
+                continue
+
+            try:
+                is_dir = member.is_dir()
+            except AttributeError:
+                is_dir = member.isdir()
+
+            if is_dir:
+                os.makedirs(dest, exist_ok=True)
+            else:
+                try:
+                    member_f = archive.open(member)
+                except TypeError:
+                    member_f = tarfile.ExFileObject(archive, member)
+
+                with open(dest, "wb") as f:
+                    f.write(member_f.read())
+                    member_f.close()
+
+        archive.close()
+
+        result["changed"] = True
+        return result

awx/playbooks/library/project_archive.py

@@ -0,0 +1,40 @@
+ANSIBLE_METADATA = {
+    "metadata_version": "1.0",
+    "status": ["stableinterface"],
+    "supported_by": "community",
+}
+
+
+DOCUMENTATION = """
+---
+module: project_archive
+short_description: unpack a project archive
+description:
+  - Unpacks an archive that contains a project, in order to support handling versioned
+    artifacts from (for example) GitHub Releases or Artifactory builds.
+  - Handles projects in the archive root, or in a single base directory of the archive.
+version_added: "2.9"
+options:
+  src:
+    description:
+    - The source archive of the project artifact
+    required: true
+  project_path:
+    description:
+    - Directory to write the project archive contents
+    required: true
+  force:
+    description:
+    - Files in the project_path will be overwritten by matching files in the archive
+    default: False
+
+author:
+    - "Philip Douglass" @philipsd6
+"""
+
+EXAMPLES = """
+- project_archive:
+    src: "{{ project_path }}/.archive/project.tar.gz"
+    project_path: "{{ project_path }}"
+    force: "{{ scm_clean }}"
+"""

awx/playbooks/project_update.yml

@@ -101,6 +101,50 @@
       tags:
         - update_insights
 
+    - block:
+        - name: Ensure the project archive directory is present
+          file:
+            dest: "{{ project_path|quote }}/.archive"
+            state: directory
+
+        - name: Get archive from url
+          get_url:
+            url: "{{ scm_url|quote }}"
+            dest: "{{ project_path|quote }}/.archive/"
+            url_username: "{{ scm_username|default(omit) }}"
+            url_password: "{{ scm_password|default(omit) }}"
+            force_basic_auth: true
+          register: get_archive
+
+        - name: Unpack archive
+          project_archive:
+            src: "{{ get_archive.dest }}"
+            project_path: "{{ project_path|quote }}"
+            force: "{{ scm_clean }}"
+          when: get_archive.changed or scm_clean
+          register: unarchived
+
+        - name: Find previous archives
+          find:
+            paths: "{{ project_path|quote }}/.archive/"
+            excludes:
+              - "{{ get_archive.dest|basename }}"
+          when: unarchived.changed
+          register: previous_archive
+
+        - name: Remove previous archives
+          file:
+            path: "{{ item.path }}"
+            state: absent
+          loop: "{{ previous_archive.files }}"
+          when: previous_archive.files|default([])
+
+        - name: Set scm_version to archive sha1 checksum
+          set_fact:
+            scm_version: "{{ get_archive.checksum_src }}"
+      tags:
+        - update_archive
+
     - name: Repository Version
       debug:
         msg: "Repository Version {{ scm_version }}"
@@ -109,6 +153,7 @@
         - update_hg
         - update_svn
         - update_insights
+        - update_archive
 
 - hosts: localhost
   gather_facts: false

awx/ui/client/src/projects/add/projects-add.controller.js

@@ -23,7 +23,7 @@ export default ['$scope', '$location', '$stateParams', 'GenerateForm',
             $scope.canEditOrg = true;
             const virtualEnvs = ConfigData.custom_virtualenvs || [];
             $scope.custom_virtualenvs_options = virtualEnvs;
-            
+
             const [ProjectModel] = resolvedModels;
             $scope.canAdd = ProjectModel.options('actions.POST');
 
@@ -170,6 +170,14 @@ export default ['$scope', '$location', '$stateParams', 'GenerateForm',
                             $scope.lookupType = 'scm_credential';
                             $scope.scmBranchLabel = i18n._('SCM Branch/Tag/Revision');
                             break;
+                        case 'archive':
+                            $scope.credentialLabel = "SCM " + i18n._("Credential");
+                            $scope.urlPopover = '<p>' + i18n._('Example URLs for Remote Archive SCM include:') + '</p>' +
+                                '<ul class=\"no-bullets\"><li>https://github.com/username/project/archive/v0.0.1.tar.gz</li>' +
+                                '<li>http://github.com/username/project/archive/v0.0.2.zip</li></ul>';
+                            $scope.credRequired = false;
+                            $scope.lookupType = 'scm_credential';
+                            break;
                         case 'insights':
                             $scope.pathRequired = false;
                             $scope.scmRequired = false;

awx/ui/client/src/projects/edit/projects-edit.controller.js

@@ -291,6 +291,14 @@ export default ['$scope', '$rootScope', '$stateParams', 'ProjectsForm', 'Rest',
                             $scope.lookupType = 'scm_credential';
                             $scope.scmBranchLabel = i18n._('SCM Branch/Tag/Revision');
                             break;
+                        case 'archive':
+                            $scope.credentialLabel = "SCM " + i18n._("Credential");
+                            $scope.urlPopover = '<p>' + i18n._('Example URLs for Remote Archive SCM include:') + '</p>' +
+                                '<ul class=\"no-bullets\"><li>https://github.com/username/project/archive/v0.0.1.tar.gz</li>' +
+                                '<li>http://github.com/username/project/archive/v0.0.2.zip</li></ul>';
+                            $scope.credRequired = false;
+                            $scope.lookupType = 'scm_credential';
+                            break;
                         case 'insights':
                             $scope.pathRequired = false;
                             $scope.scmRequired = false;

awx/ui/client/src/projects/projects.form.js

@@ -124,7 +124,7 @@ export default ['i18n', 'NotificationsList', 'TemplateList',
             scm_branch: {
                 labelBind: "scmBranchLabel",
                 type: 'text',
-                ngShow: "scm_type && scm_type.value !== 'manual' && scm_type.value !== 'insights'",
+                ngShow: "scm_type && scm_type.value !== 'manual' && scm_type.value !== 'insights' && scm_type.value !== 'archive'",
                 ngDisabled: '!(project_obj.summary_fields.user_capabilities.edit || canAdd)',
                 awPopOver: '<p>' + i18n._("Branch to checkout.  In addition to branches, you can input tags, commit hashes, and arbitrary refs.  Some commit hashes and refs may not be availble unless you also provide a custom refspec.") + '</p>',
                 dataTitle: i18n._('SCM Branch'),

awx/ui_next/src/components/Lookup/ProjectLookup.jsx

@@ -105,6 +105,7 @@ function ProjectLookup({
                   [`git`, i18n._(t`Git`)],
                   [`hg`, i18n._(t`Mercurial`)],
                   [`svn`, i18n._(t`Subversion`)],
+                  [`archive`, i18n._(t`Remote Archive`)],
                   [`insights`, i18n._(t`Red Hat Insights`)],
                 ],
               },

awx/ui_next/src/components/UserAndTeamAccessAdd/getResourceAccessConfig.js

@@ -89,6 +89,7 @@ export default function getResourceAccessConfig(i18n) {
             [`git`, i18n._(t`Git`)],
             [`hg`, i18n._(t`Mercurial`)],
             [`svn`, i18n._(t`Subversion`)],
+            [`archive`, i18n._(t`Remote Archive`)],
             [`insights`, i18n._(t`Red Hat Insights`)],
           ],
         },
@@ -158,6 +159,7 @@ export default function getResourceAccessConfig(i18n) {
             [`git`, i18n._(t`Git`)],
             [`hg`, i18n._(t`Mercurial`)],
             [`svn`, i18n._(t`Subversion`)],
+            [`archive`, i18n._(t`Remote Archive`)],
             [`insights`, i18n._(t`Red Hat Insights`)],
           ],
         },

awx/ui_next/src/screens/Project/ProjectAdd/ProjectAdd.test.jsx

@@ -37,6 +37,7 @@ describe('<ProjectAdd />', () => {
               ['git', 'Git'],
               ['hg', 'Mercurial'],
               ['svn', 'Subversion'],
+              ['archive', 'Remote Archive'],
               ['insights', 'Red Hat Insights'],
             ],
           },

awx/ui_next/src/screens/Project/ProjectEdit/ProjectEdit.test.jsx

@@ -49,6 +49,7 @@ describe('<ProjectEdit />', () => {
               ['git', 'Git'],
               ['hg', 'Mercurial'],
               ['svn', 'Subversion'],
+              ['archive', 'Remote Archive'],
               ['insights', 'Red Hat Insights'],
             ],
           },

awx/ui_next/src/screens/Project/ProjectList/ProjectList.jsx

@@ -138,6 +138,7 @@ function ProjectList({ i18n }) {
                   [`git`, i18n._(t`Git`)],
                   [`hg`, i18n._(t`Mercurial`)],
                   [`svn`, i18n._(t`Subversion`)],
+                  [`archive`, i18n._(t`Remote Archive`)],
                   [`insights`, i18n._(t`Red Hat Insights`)],
                 ],
               },

awx/ui_next/src/screens/Project/ProjectList/ProjectList.test.jsx

@@ -61,6 +61,24 @@ const mockProjects = [
       },
     },
   },
+  {
+    id: 4,
+    name: 'Project 4',
+    url: '/api/v2/projects/4',
+    type: 'project',
+    scm_type: 'archive',
+    scm_revision: 'odsd9ajf8aagjisooajfij34ikdj3fs994s4daiaos7',
+    summary_fields: {
+      last_job: {
+        id: 9004,
+        status: 'successful',
+      },
+      user_capabilities: {
+        delete: false,
+        update: false,
+      },
+    },
+  },
 ];
 
 describe('<ProjectList />', () => {
@@ -94,7 +112,7 @@ describe('<ProjectList />', () => {
     });
     wrapper.update();
 
-    expect(wrapper.find('ProjectListItem')).toHaveLength(3);
+    expect(wrapper.find('ProjectListItem')).toHaveLength(4);
   });
 
   test('should select project when checked', async () => {
@@ -133,7 +151,7 @@ describe('<ProjectList />', () => {
     wrapper.update();
 
     const items = wrapper.find('ProjectListItem');
-    expect(items).toHaveLength(3);
+    expect(items).toHaveLength(4);
     items.forEach(item => {
       expect(item.prop('isSelected')).toEqual(true);
     });

awx/ui_next/src/screens/Project/shared/ProjectForm.jsx

@@ -25,6 +25,7 @@ import {
   GitSubForm,
   HgSubForm,
   SvnSubForm,
+  ArchiveSubForm,
   InsightsSubForm,
   ManualSubForm,
 } from './ProjectSubForms';
@@ -240,6 +241,13 @@ function ProjectFormFields({
                     scmUpdateOnLaunch={formik.values.scm_update_on_launch}
                   />
                 ),
+                archive: (
+                  <ArchiveSubForm
+                    credential={credentials.scm}
+                    onCredentialSelection={handleCredentialSelection}
+                    scmUpdateOnLaunch={formik.values.scm_update_on_launch}
+                  />
+                ),
                 insights: (
                   <InsightsSubForm
                     credential={credentials.insights}

awx/ui_next/src/screens/Project/shared/ProjectForm.test.jsx

@@ -47,6 +47,7 @@ describe('<ProjectForm />', () => {
               ['git', 'Git'],
               ['hg', 'Mercurial'],
               ['svn', 'Subversion'],
+              ['archive', 'Remote Archive'],
               ['insights', 'Red Hat Insights'],
             ],
           },

awx/ui_next/src/screens/Project/shared/ProjectSubForms/ArchiveSubForm.jsx

@@ -0,0 +1,38 @@
+import 'styled-components/macro';
+import React from 'react';
+import { withI18n } from '@lingui/react';
+import { t } from '@lingui/macro';
+import {
+  UrlFormField,
+  ScmCredentialFormField,
+  ScmTypeOptions,
+} from './SharedFields';
+
+const ArchiveSubForm = ({
+  i18n,
+  credential,
+  onCredentialSelection,
+  scmUpdateOnLaunch,
+}) => (
+  <>
+    <UrlFormField
+      i18n={i18n}
+      tooltip={
+        <span>
+          {i18n._(t`Example URLs for Remote Archive Source Control include:`)}
+          <ul css={{ margin: '10px 0 10px 20px' }}>
+            <li>https://github.com/username/project/archive/v0.0.1.tar.gz</li>
+            <li>https://github.com/username/project/archive/v0.0.2.zip</li>
+          </ul>
+        </span>
+      }
+    />
+    <ScmCredentialFormField
+      credential={credential}
+      onCredentialSelection={onCredentialSelection}
+    />
+    <ScmTypeOptions scmUpdateOnLaunch={scmUpdateOnLaunch} />
+  </>
+);
+
+export default withI18n()(ArchiveSubForm);

awx/ui_next/src/screens/Project/shared/ProjectSubForms/index.js

@@ -3,3 +3,4 @@ export { default as HgSubForm } from './HgSubForm';
 export { default as InsightsSubForm } from './InsightsSubForm';
 export { default as ManualSubForm } from './ManualSubForm';
 export { default as SvnSubForm } from './SvnSubForm';
+export { default as ArchiveSubForm } from './ArchiveSubForm';

awx/ui_next/src/screens/Template/WorkflowJobTemplateVisualizer/Modals/NodeModals/NodeTypeStep/ProjectsList.jsx

@@ -90,6 +90,7 @@ function ProjectsList({ i18n, nodeResource, onUpdateNodeResource }) {
             [`git`, i18n._(t`Git`)],
             [`hg`, i18n._(t`Mercurial`)],
             [`svn`, i18n._(t`Subversion`)],
+            [`archive`, i18n._(t`Remote Archive`)],
             [`insights`, i18n._(t`Red Hat Insights`)],
           ],
         },

awx/ui_next/src/types.js

@@ -148,7 +148,7 @@ export const Project = shape({
   created: string,
   name: string.isRequired,
   description: string,
-  scm_type: oneOf(['', 'git', 'hg', 'svn', 'insights']),
+  scm_type: oneOf(['', 'git', 'hg', 'svn', 'archive', 'insights']),
   scm_url: string,
   scm_branch: string,
   scm_refspec: string,