From 2cfa4eb60a190565770f14d7749381aa8a5d8520 Mon Sep 17 00:00:00 2001 From: Philip Douglass Date: Fri, 6 Dec 2019 13:18:02 -0500 Subject: [PATCH 01/14] Add archive option to SCM_TYPE_CHOICES for Remote Archives Signed-off-by: Philip Douglass --- awx/main/models/projects.py | 1 + 1 file changed, 1 insertion(+) diff --git a/awx/main/models/projects.py b/awx/main/models/projects.py index 5d8cfd5290..bc52d4269c 100644 --- a/awx/main/models/projects.py +++ b/awx/main/models/projects.py @@ -55,6 +55,7 @@ class ProjectOptions(models.Model): ('hg', _('Mercurial')), ('svn', _('Subversion')), ('insights', _('Red Hat Insights')), + ('archive', _('Remote Archive')), ] class Meta: From 47cabc42290f5df77eb280bf4ef74e4d8c8f1fed Mon Sep 17 00:00:00 2001 From: Philip Douglass Date: Tue, 30 Jul 2019 13:32:25 -0400 Subject: [PATCH 02/14] Add archive SCM url handling to update_scm_url() Signed-off-by: Philip Douglass --- awx/main/utils/common.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/awx/main/utils/common.py b/awx/main/utils/common.py index 75fcd306ef..f34cf4e4d8 100644 --- a/awx/main/utils/common.py +++ b/awx/main/utils/common.py @@ -257,7 +257,7 @@ def update_scm_url(scm_type, url, username=True, password=True, # git: https://www.kernel.org/pub/software/scm/git/docs/git-clone.html#URLS # hg: http://www.selenic.com/mercurial/hg.1.html#url-paths # svn: http://svnbook.red-bean.com/en/1.7/svn-book.html#svn.advanced.reposurls - if scm_type not in ('git', 'hg', 'svn', 'insights'): + if scm_type not in ('git', 'hg', 'svn', 'insights', 'archive'): raise ValueError(_('Unsupported SCM type "%s"') % str(scm_type)) if not url.strip(): return '' @@ -303,7 +303,8 @@ def update_scm_url(scm_type, url, username=True, password=True, 'git': ('ssh', 'git', 'git+ssh', 'http', 'https', 'ftp', 'ftps', 'file'), 'hg': ('http', 'https', 'ssh', 'file'), 'svn': ('http', 'https', 'svn', 'svn+ssh', 'file'), - 'insights': ('http', 'https') + 'insights': ('http', 'https'), + 'archive': ('http', 'https'), } if parts.scheme not in scm_type_schemes.get(scm_type, ()): raise ValueError(_('Unsupported %s URL') % scm_type) @@ -339,7 +340,7 @@ def update_scm_url(scm_type, url, username=True, password=True, #raise ValueError('Password not supported for SSH with Mercurial.') netloc_password = '' - if netloc_username and parts.scheme != 'file' and scm_type != "insights": + if netloc_username and parts.scheme != 'file' and scm_type not in ("insights", "archive"): netloc = u':'.join([urllib.parse.quote(x,safe='') for x in (netloc_username, netloc_password) if x]) else: netloc = u'' From 997351eee3d831f394310aa5712bc090cf840846 Mon Sep 17 00:00:00 2001 From: Philip Douglass Date: Tue, 30 Jul 2019 13:43:47 -0400 Subject: [PATCH 03/14] Add archive project data to Dashboard view Signed-off-by: Philip Douglass --- awx/api/views/__init__.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/awx/api/views/__init__.py b/awx/api/views/__init__.py index f6378f5282..c5b22d105a 100644 --- a/awx/api/views/__init__.py +++ b/awx/api/views/__init__.py @@ -242,6 +242,8 @@ class DashboardView(APIView): svn_failed_projects = svn_projects.filter(last_job_failed=True) hg_projects = user_projects.filter(scm_type='hg') hg_failed_projects = hg_projects.filter(last_job_failed=True) + archive_projects = user_projects.filter(scm_type='archive') + archive_failed_projects = archive_projects.filter(last_job_failed=True) data['scm_types'] = {} data['scm_types']['git'] = {'url': reverse('api:project_list', request=request) + "?scm_type=git", 'label': 'Git', @@ -258,6 +260,11 @@ class DashboardView(APIView): 'failures_url': reverse('api:project_list', request=request) + "?scm_type=hg&last_job_failed=True", 'total': hg_projects.count(), 'failed': hg_failed_projects.count()} + data['scm_types']['archive'] = {'url': reverse('api:project_list', request=request) + "?scm_type=archive", + 'label': 'Remote Archive', + 'failures_url': reverse('api:project_list', request=request) + "?scm_type=archive&last_job_failed=True", + 'total': archive_projects.count(), + 'failed': archive_failed_projects.count()} user_list = get_user_queryset(request.user, models.User) team_list = get_user_queryset(request.user, models.Team) From d224aa09f0bd908944e9be778867f018149b3974 Mon Sep 17 00:00:00 2001 From: Philip Douglass Date: Tue, 30 Jul 2019 13:49:27 -0400 Subject: [PATCH 04/14] Add archive to TestProjectUpdateCredentials test parametrize Signed-off-by: Philip Douglass --- awx/main/tests/unit/test_tasks.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/awx/main/tests/unit/test_tasks.py b/awx/main/tests/unit/test_tasks.py index 71bcd8d03c..7dcea33ccd 100644 --- a/awx/main/tests/unit/test_tasks.py +++ b/awx/main/tests/unit/test_tasks.py @@ -1792,16 +1792,19 @@ class TestProjectUpdateCredentials(TestJobExecution): dict(scm_type='git'), dict(scm_type='hg'), dict(scm_type='svn'), + dict(scm_type='archive'), ], 'test_ssh_key_auth': [ dict(scm_type='git'), dict(scm_type='hg'), dict(scm_type='svn'), + dict(scm_type='archive'), ], 'test_awx_task_env': [ dict(scm_type='git'), dict(scm_type='hg'), dict(scm_type='svn'), + dict(scm_type='archive'), ] } From 2f3f6e60d1f3f084748f1c13580e6cc138eb8649 Mon Sep 17 00:00:00 2001 From: Philip Douglass Date: Tue, 30 Jul 2019 13:52:08 -0400 Subject: [PATCH 05/14] Add archive to scm_type expected test results Signed-off-by: Philip Douglass --- .../tests/functional/analytics/test_projects_by_scm_type.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/awx/main/tests/functional/analytics/test_projects_by_scm_type.py b/awx/main/tests/functional/analytics/test_projects_by_scm_type.py index 29ffbd6283..1590b5d3bb 100644 --- a/awx/main/tests/functional/analytics/test_projects_by_scm_type.py +++ b/awx/main/tests/functional/analytics/test_projects_by_scm_type.py @@ -12,7 +12,8 @@ def test_empty(): 'git': 0, 'svn': 0, 'hg': 0, - 'insights': 0 + 'insights': 0, + 'archive': 0, } @@ -24,7 +25,8 @@ def test_multiple(scm_type): 'git': 0, 'svn': 0, 'hg': 0, - 'insights': 0 + 'insights': 0, + 'archive': 0, } for i in range(random.randint(0, 10)): Project(scm_type=scm_type).save() From 8157ab2fa94923fd149a8ab45aa1c8f6e2d9fc4f Mon Sep 17 00:00:00 2001 From: Philip Douglass Date: Tue, 30 Jul 2019 16:29:57 -0400 Subject: [PATCH 06/14] Hide scm_branch field when scm_type is archive Signed-off-by: Philip Douglass --- awx/ui/client/src/projects/projects.form.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/awx/ui/client/src/projects/projects.form.js b/awx/ui/client/src/projects/projects.form.js index 13ef01d2f0..dac0118c03 100644 --- a/awx/ui/client/src/projects/projects.form.js +++ b/awx/ui/client/src/projects/projects.form.js @@ -124,7 +124,7 @@ export default ['i18n', 'NotificationsList', 'TemplateList', scm_branch: { labelBind: "scmBranchLabel", type: 'text', - ngShow: "scm_type && scm_type.value !== 'manual' && scm_type.value !== 'insights'", + ngShow: "scm_type && scm_type.value !== 'manual' && scm_type.value !== 'insights' && scm_type.value !== 'archive'", ngDisabled: '!(project_obj.summary_fields.user_capabilities.edit || canAdd)', awPopOver: '

' + i18n._("Branch to checkout. In addition to branches, you can input tags, commit hashes, and arbitrary refs. Some commit hashes and refs may not be availble unless you also provide a custom refspec.") + '

', dataTitle: i18n._('SCM Branch'), From b011e34faeb65d88ef49b1b7400562a48a6f92bc Mon Sep 17 00:00:00 2001 From: Philip Douglass Date: Tue, 30 Jul 2019 16:34:49 -0400 Subject: [PATCH 07/14] Show fields for archive when selected as scm_type Signed-off-by: Philip Douglass --- .../client/src/projects/add/projects-add.controller.js | 10 +++++++++- .../src/projects/edit/projects-edit.controller.js | 8 ++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/awx/ui/client/src/projects/add/projects-add.controller.js b/awx/ui/client/src/projects/add/projects-add.controller.js index c34a6ecdf2..952cb07974 100644 --- a/awx/ui/client/src/projects/add/projects-add.controller.js +++ b/awx/ui/client/src/projects/add/projects-add.controller.js @@ -23,7 +23,7 @@ export default ['$scope', '$location', '$stateParams', 'GenerateForm', $scope.canEditOrg = true; const virtualEnvs = ConfigData.custom_virtualenvs || []; $scope.custom_virtualenvs_options = virtualEnvs; - + const [ProjectModel] = resolvedModels; $scope.canAdd = ProjectModel.options('actions.POST'); @@ -170,6 +170,14 @@ export default ['$scope', '$location', '$stateParams', 'GenerateForm', $scope.lookupType = 'scm_credential'; $scope.scmBranchLabel = i18n._('SCM Branch/Tag/Revision'); break; + case 'archive': + $scope.credentialLabel = "SCM " + i18n._("Credential"); + $scope.urlPopover = '

' + i18n._('Example URLs for Remote Archive SCM include:') + '

' + + '
  • https://github.com/username/project/archive/v0.0.1.tar.gz
    • ' + + '
  • http://github.com/username/project/archive/v0.0.2.zip
'; + $scope.credRequired = false; + $scope.lookupType = 'scm_credential'; + break; case 'insights': $scope.pathRequired = false; $scope.scmRequired = false; diff --git a/awx/ui/client/src/projects/edit/projects-edit.controller.js b/awx/ui/client/src/projects/edit/projects-edit.controller.js index 27f6d3b376..8215bbab47 100644 --- a/awx/ui/client/src/projects/edit/projects-edit.controller.js +++ b/awx/ui/client/src/projects/edit/projects-edit.controller.js @@ -291,6 +291,14 @@ export default ['$scope', '$rootScope', '$stateParams', 'ProjectsForm', 'Rest', $scope.lookupType = 'scm_credential'; $scope.scmBranchLabel = i18n._('SCM Branch/Tag/Revision'); break; + case 'archive': + $scope.credentialLabel = "SCM " + i18n._("Credential"); + $scope.urlPopover = '

' + i18n._('Example URLs for Remote Archive SCM include:') + '

' + + '
  • https://github.com/username/project/archive/v0.0.1.tar.gz
    • ' + + '
  • http://github.com/username/project/archive/v0.0.2.zip
'; + $scope.credRequired = false; + $scope.lookupType = 'scm_credential'; + break; case 'insights': $scope.pathRequired = false; $scope.scmRequired = false; From 6720cd9bda2bd11da2e2661cf777483f767225d0 Mon Sep 17 00:00:00 2001 From: Philip Douglass Date: Tue, 30 Jul 2019 22:34:14 -0400 Subject: [PATCH 08/14] Add block to download and unpack a remote archive Signed-off-by: Philip Douglass --- awx/playbooks/project_update.yml | 60 ++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/awx/playbooks/project_update.yml b/awx/playbooks/project_update.yml index e572496497..22478f9f0c 100644 --- a/awx/playbooks/project_update.yml +++ b/awx/playbooks/project_update.yml @@ -101,6 +101,65 @@ tags: - update_insights + - block: + - name: Ensure the project directory is present + file: + dest: "{{ project_path|quote }}" + state: directory + + - name: Get archive from url + get_url: + url: "{{ scm_url|quote }}" + dest: "{{ project_path|quote }}" + url_username: "{{ scm_username|default(omit) }}" + url_password: "{{ scm_password|default(omit) }}" + force_basic_auth: yes + force: "{{ scm_clean }}" + register: get_url_result + + - name: Unpack archive + unarchive: + src: "{{ get_url_result.dest }}" + dest: "{{ project_path|quote }}" + keep_newer: "{{ not scm_clean }}" + list_files: yes + register: unarchived + + - set_fact: + archive_root_dirs: "{{ archive_root_dirs |default([]) |union(item.split('/')[0:1]) }}" + loop: "{{ unarchived.files }}" + + - block: + - name: Delete unarchived single root directory + file: + dest: "{{ project_path|quote }}/{{ archive_root_dirs[0] }}" + state: absent + + - name: Link single root directory to project directory + file: + src: "{{ project_path|quote }}" + dest: "{{ project_path|quote }}/{{ archive_root_dirs[0] }}" + state: link + + - name: Unpack archive + unarchive: + src: "{{ get_url_result.dest }}" + dest: "{{ project_path|quote }}" + keep_newer: "{{ not scm_clean }}" + + - name: Delete link + file: + dest: "{{ project_path|quote }}/{{ archive_root_dirs[0] }}" + state: absent + when: archive_root_dirs |length == 1 + + - name: Set scm_version to archive sha1 checksum + set_fact: + scm_version: "{{ get_url_result.checksum_src }}" + + tags: + - update_archive + - name: Repository Version debug: msg: "Repository Version {{ scm_version }}" @@ -109,6 +168,7 @@ - update_hg - update_svn - update_insights + - update_archive - hosts: localhost gather_facts: false From f72b777b079dbe060b2f2b8cceb1b3336974106c Mon Sep 17 00:00:00 2001 From: Philip DOUGLASS Date: Sun, 16 Aug 2020 14:47:23 -0400 Subject: [PATCH 09/14] Add plugin to cleanly manage possibly versioned project archives Signed-off-by: Philip Douglass --- .../action_plugins/project_archive.py | 82 ++++++++++++++++++ awx/playbooks/library/project_archive.py | 40 +++++++++ awx/playbooks/project_update.yml | 85 ++++++++----------- 3 files changed, 157 insertions(+), 50 deletions(-) create mode 100644 awx/playbooks/action_plugins/project_archive.py create mode 100644 awx/playbooks/library/project_archive.py diff --git a/awx/playbooks/action_plugins/project_archive.py b/awx/playbooks/action_plugins/project_archive.py new file mode 100644 index 0000000000..753650d12b --- /dev/null +++ b/awx/playbooks/action_plugins/project_archive.py @@ -0,0 +1,82 @@ +from __future__ import absolute_import, division, print_function + +__metaclass__ = type + +import zipfile +import tarfile +import os + +from ansible.plugins.action import ActionBase +from ansible.utils.display import Display + +display = Display() + + +class ActionModule(ActionBase): + def run(self, tmp=None, task_vars=None): + self._supports_check_mode = False + + result = super(ActionModule, self).run(tmp, task_vars) + + src = self._task.args.get("src") + proj_path = self._task.args.get("project_path") + force = self._task.args.get("force", False) + + try: + archive = zipfile.ZipFile(src) + get_filenames = archive.namelist + get_members = archive.infolist + except zipfile.BadZipFile: + archive = tarfile.open(src) + get_filenames = archive.getnames + get_members = archive.getmembers + except tarfile.ReadError: + result["failed"] = True + result["msg"] = "{0} is not a valid archive".format(src) + return result + + # Most well formed archives contain a single root directory, typically named + # project-name-1.0.0. The project contents should be inside that directory. + start_index = 0 + root_contents = set( + [filename.split(os.path.sep)[0] for filename in get_filenames()] + ) + if len(root_contents) == 1: + start_index = len(list(root_contents)[0]) + 1 + + for member in get_members(): + try: + filename = member.filename + except AttributeError: + filename = member.name + + # Skip the archive base directory + if not filename[start_index:]: + continue + + dest = os.path.join(proj_path, filename[start_index:]) + + if not force and os.path.exists(dest): + continue + + try: + is_dir = member.is_dir() + except AttributeError: + is_dir = member.isdir() + + if is_dir: + os.makedirs(dest, exist_ok=True) + else: + try: + member_f = archive.open(member) + except TypeError: + member_f = tarfile.ExFileObject(archive, member) + + with open(dest, "wb") as f: + f.write(member_f.read()) + member_f.close() + + archive.close() + + result["changed"] = True + return result diff --git a/awx/playbooks/library/project_archive.py b/awx/playbooks/library/project_archive.py new file mode 100644 index 0000000000..4a046e354d --- /dev/null +++ b/awx/playbooks/library/project_archive.py @@ -0,0 +1,40 @@ +ANSIBLE_METADATA = { + "metadata_version": "1.0", + "status": ["stableinterface"], + "supported_by": "community", +} + + +DOCUMENTATION = """ +--- +module: project_archive +short_description: unpack a project archive +description: + - Unpacks an archive that contains a project, in order to support handling versioned + artifacts from (for example) GitHub Releases or Artifactory builds. + - Handles projects in the archive root, or in a single base directory of the archive. +version_added: "2.9" +options: + src: + description: + - The source archive of the project artifact + required: true + project_path: + description: + - Directory to write the project archive contents + required: true + force: + description: + - Files in the project_path will be overwritten by matching files in the archive + default: False + +author: + - "Philip Douglass" @philipsd6 +""" + +EXAMPLES = """ +- project_archive: + src: "{{ project_path }}/.archive/project.tar.gz" + project_path: "{{ project_path }}" + force: "{{ scm_clean }}" +""" diff --git a/awx/playbooks/project_update.yml b/awx/playbooks/project_update.yml index 22478f9f0c..169273d628 100644 --- a/awx/playbooks/project_update.yml +++ b/awx/playbooks/project_update.yml @@ -102,61 +102,46 @@ - update_insights - block: - - name: Ensure the project directory is present - file: - dest: "{{ project_path|quote }}" - state: directory + - name: Ensure the project archive directory is present + file: + dest: "{{ project_path|quote }}/.archive" + state: directory - - name: Get archive from url - get_url: - url: "{{ scm_url|quote }}" - dest: "{{ project_path|quote }}" - url_username: "{{ scm_username|default(omit) }}" - url_password: "{{ scm_password|default(omit) }}" - force_basic_auth: yes - force: "{{ scm_clean }}" - register: get_url_result + - name: Get archive from url + get_url: + url: "{{ scm_url|quote }}" + dest: "{{ project_path|quote }}/.archive/" + url_username: "{{ scm_username|default(omit) }}" + url_password: "{{ scm_password|default(omit) }}" + force_basic_auth: true + register: get_archive - - name: Unpack archive - unarchive: - src: "{{ get_url_result.dest }}" - dest: "{{ project_path|quote }}" - keep_newer: "{{ not scm_clean }}" - list_files: yes - register: unarchived + - name: Unpack archive + project_archive: + src: "{{ get_archive.dest }}" + project_path: "{{ project_path|quote }}" + force: "{{ scm_clean }}" + when: get_archive.changed or scm_clean + register: unarchived - - set_fact: - archive_root_dirs: "{{ archive_root_dirs |default([]) |union(item.split('/')[0:1]) }}" - loop: "{{ unarchived.files }}" + - name: Find previous archives + find: + paths: "{{ project_path|quote }}/.archive/" + excludes: + - "{{ get_archive.dest|basename }}" + when: unarchived.changed + register: previous_archive - - block: - - name: Delete unarchived single root directory - file: - dest: "{{ project_path|quote }}/{{ archive_root_dirs[0] }}" - state: absent - - - name: Link single root directory to project directory - file: - src: "{{ project_path|quote }}" - dest: "{{ project_path|quote }}/{{ archive_root_dirs[0] }}" - state: link - - - name: Unpack archive - unarchive: - src: "{{ get_url_result.dest }}" - dest: "{{ project_path|quote }}" - keep_newer: "{{ not scm_clean }}" - - - name: Delete link - file: - dest: "{{ project_path|quote }}/{{ archive_root_dirs[0] }}" - state: absent - when: archive_root_dirs |length == 1 - - - name: Set scm_version to archive sha1 checksum - set_fact: - scm_version: "{{ get_url_result.checksum_src }}" + - name: Remove previous archives + file: + path: "{{ item.path }}" + state: absent + loop: "{{ previous_archive.files }}" + when: previous_archive.files|default([]) + - name: Set scm_version to archive sha1 checksum + set_fact: + scm_version: "{{ get_archive.checksum_src }}" tags: - update_archive From 70cbccd2ef281b954f1c971adbd938ed98282fe8 Mon Sep 17 00:00:00 2001 From: Philip Douglass Date: Tue, 18 Aug 2020 18:42:11 -0400 Subject: [PATCH 10/14] Add migration for new Remote Archive SCM Type --- .../0118_add_remote_archive_scm_type.py | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 awx/main/migrations/0118_add_remote_archive_scm_type.py diff --git a/awx/main/migrations/0118_add_remote_archive_scm_type.py b/awx/main/migrations/0118_add_remote_archive_scm_type.py new file mode 100644 index 0000000000..246ca4c823 --- /dev/null +++ b/awx/main/migrations/0118_add_remote_archive_scm_type.py @@ -0,0 +1,23 @@ +# Generated by Django 2.2.11 on 2020-08-18 22:35 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('main', '0117_v400_remove_cloudforms_inventory'), + ] + + operations = [ + migrations.AlterField( + model_name='project', + name='scm_type', + field=models.CharField(blank=True, choices=[('', 'Manual'), ('git', 'Git'), ('hg', 'Mercurial'), ('svn', 'Subversion'), ('insights', 'Red Hat Insights'), ('archive', 'Remote Archive')], default='', help_text='Specifies the source control system used to store the project.', max_length=8, verbose_name='SCM Type'), + ), + migrations.AlterField( + model_name='projectupdate', + name='scm_type', + field=models.CharField(blank=True, choices=[('', 'Manual'), ('git', 'Git'), ('hg', 'Mercurial'), ('svn', 'Subversion'), ('insights', 'Red Hat Insights'), ('archive', 'Remote Archive')], default='', help_text='Specifies the source control system used to store the project.', max_length=8, verbose_name='SCM Type'), + ), + ] From d2595003329dbc79faaddb7e331471ec27cb3a3c Mon Sep 17 00:00:00 2001 From: Philip Douglass Date: Wed, 19 Aug 2020 18:55:13 -0400 Subject: [PATCH 11/14] Implement Remote Archive SCM Type for ui_next Signed-off-by: Philip Douglass --- .../src/components/Lookup/ProjectLookup.jsx | 1 + .../getResourceAccessConfig.js | 2 + .../Project/ProjectAdd/ProjectAdd.test.jsx | 1 + .../Project/ProjectEdit/ProjectEdit.test.jsx | 1 + .../Project/ProjectList/ProjectList.jsx | 1 + .../Project/ProjectList/ProjectList.test.jsx | 22 ++++++++++- .../screens/Project/shared/ProjectForm.jsx | 8 ++++ .../Project/shared/ProjectForm.test.jsx | 1 + .../shared/ProjectSubForms/ArchiveSubForm.jsx | 38 +++++++++++++++++++ .../Project/shared/ProjectSubForms/index.js | 1 + .../NodeModals/NodeTypeStep/ProjectsList.jsx | 1 + awx/ui_next/src/types.js | 2 +- 12 files changed, 76 insertions(+), 3 deletions(-) create mode 100644 awx/ui_next/src/screens/Project/shared/ProjectSubForms/ArchiveSubForm.jsx diff --git a/awx/ui_next/src/components/Lookup/ProjectLookup.jsx b/awx/ui_next/src/components/Lookup/ProjectLookup.jsx index dcc16669b8..5c8ec16dee 100644 --- a/awx/ui_next/src/components/Lookup/ProjectLookup.jsx +++ b/awx/ui_next/src/components/Lookup/ProjectLookup.jsx @@ -105,6 +105,7 @@ function ProjectLookup({ [`git`, i18n._(t`Git`)], [`hg`, i18n._(t`Mercurial`)], [`svn`, i18n._(t`Subversion`)], + [`archive`, i18n._(t`Remote Archive`)], [`insights`, i18n._(t`Red Hat Insights`)], ], }, diff --git a/awx/ui_next/src/components/UserAndTeamAccessAdd/getResourceAccessConfig.js b/awx/ui_next/src/components/UserAndTeamAccessAdd/getResourceAccessConfig.js index cd922c23aa..fd19a58299 100644 --- a/awx/ui_next/src/components/UserAndTeamAccessAdd/getResourceAccessConfig.js +++ b/awx/ui_next/src/components/UserAndTeamAccessAdd/getResourceAccessConfig.js @@ -87,6 +87,7 @@ export default function getResourceAccessConfig(i18n) { [`git`, i18n._(t`Git`)], [`hg`, i18n._(t`Mercurial`)], [`svn`, i18n._(t`Subversion`)], + [`archive`, i18n._(t`Remote Archive`)], [`insights`, i18n._(t`Red Hat Insights`)], ], }, @@ -154,6 +155,7 @@ export default function getResourceAccessConfig(i18n) { [`git`, i18n._(t`Git`)], [`hg`, i18n._(t`Mercurial`)], [`svn`, i18n._(t`Subversion`)], + [`archive`, i18n._(t`Remote Archive`)], [`insights`, i18n._(t`Red Hat Insights`)], ], }, diff --git a/awx/ui_next/src/screens/Project/ProjectAdd/ProjectAdd.test.jsx b/awx/ui_next/src/screens/Project/ProjectAdd/ProjectAdd.test.jsx index a47bb424bf..cb2c16a23f 100644 --- a/awx/ui_next/src/screens/Project/ProjectAdd/ProjectAdd.test.jsx +++ b/awx/ui_next/src/screens/Project/ProjectAdd/ProjectAdd.test.jsx @@ -37,6 +37,7 @@ describe('', () => { ['git', 'Git'], ['hg', 'Mercurial'], ['svn', 'Subversion'], + ['archive', 'Remote Archive'], ['insights', 'Red Hat Insights'], ], }, diff --git a/awx/ui_next/src/screens/Project/ProjectEdit/ProjectEdit.test.jsx b/awx/ui_next/src/screens/Project/ProjectEdit/ProjectEdit.test.jsx index 5b75396d65..5171e0f532 100644 --- a/awx/ui_next/src/screens/Project/ProjectEdit/ProjectEdit.test.jsx +++ b/awx/ui_next/src/screens/Project/ProjectEdit/ProjectEdit.test.jsx @@ -49,6 +49,7 @@ describe('', () => { ['git', 'Git'], ['hg', 'Mercurial'], ['svn', 'Subversion'], + ['archive', 'Remote Archive'], ['insights', 'Red Hat Insights'], ], }, diff --git a/awx/ui_next/src/screens/Project/ProjectList/ProjectList.jsx b/awx/ui_next/src/screens/Project/ProjectList/ProjectList.jsx index 7afbe124b3..b7b8661c1b 100644 --- a/awx/ui_next/src/screens/Project/ProjectList/ProjectList.jsx +++ b/awx/ui_next/src/screens/Project/ProjectList/ProjectList.jsx @@ -138,6 +138,7 @@ function ProjectList({ i18n }) { [`git`, i18n._(t`Git`)], [`hg`, i18n._(t`Mercurial`)], [`svn`, i18n._(t`Subversion`)], + [`archive`, i18n._(t`Remote Archive`)], [`insights`, i18n._(t`Red Hat Insights`)], ], }, diff --git a/awx/ui_next/src/screens/Project/ProjectList/ProjectList.test.jsx b/awx/ui_next/src/screens/Project/ProjectList/ProjectList.test.jsx index b50e569f18..5a6945d892 100644 --- a/awx/ui_next/src/screens/Project/ProjectList/ProjectList.test.jsx +++ b/awx/ui_next/src/screens/Project/ProjectList/ProjectList.test.jsx @@ -61,6 +61,24 @@ const mockProjects = [ }, }, }, + { + id: 4, + name: 'Project 4', + url: '/api/v2/projects/4', + type: 'project', + scm_type: 'archive', + scm_revision: 'odsd9ajf8aagjisooajfij34ikdj3fs994s4daiaos7', + summary_fields: { + last_job: { + id: 9004, + status: 'successful', + }, + user_capabilities: { + delete: false, + update: false, + }, + }, + }, ]; describe('', () => { @@ -94,7 +112,7 @@ describe('', () => { }); wrapper.update(); - expect(wrapper.find('ProjectListItem')).toHaveLength(3); + expect(wrapper.find('ProjectListItem')).toHaveLength(4); }); test('should select project when checked', async () => { @@ -133,7 +151,7 @@ describe('', () => { wrapper.update(); const items = wrapper.find('ProjectListItem'); - expect(items).toHaveLength(3); + expect(items).toHaveLength(4); items.forEach(item => { expect(item.prop('isSelected')).toEqual(true); }); diff --git a/awx/ui_next/src/screens/Project/shared/ProjectForm.jsx b/awx/ui_next/src/screens/Project/shared/ProjectForm.jsx index 56c740d73d..5a9f7b1ea7 100644 --- a/awx/ui_next/src/screens/Project/shared/ProjectForm.jsx +++ b/awx/ui_next/src/screens/Project/shared/ProjectForm.jsx @@ -25,6 +25,7 @@ import { GitSubForm, HgSubForm, SvnSubForm, + ArchiveSubForm, InsightsSubForm, ManualSubForm, } from './ProjectSubForms'; @@ -240,6 +241,13 @@ function ProjectFormFields({ scmUpdateOnLaunch={formik.values.scm_update_on_launch} /> ), + archive: ( + + ), insights: ( ', () => { ['git', 'Git'], ['hg', 'Mercurial'], ['svn', 'Subversion'], + ['archive', 'Remote Archive'], ['insights', 'Red Hat Insights'], ], }, diff --git a/awx/ui_next/src/screens/Project/shared/ProjectSubForms/ArchiveSubForm.jsx b/awx/ui_next/src/screens/Project/shared/ProjectSubForms/ArchiveSubForm.jsx new file mode 100644 index 0000000000..ba65b0b6ff --- /dev/null +++ b/awx/ui_next/src/screens/Project/shared/ProjectSubForms/ArchiveSubForm.jsx @@ -0,0 +1,38 @@ +import 'styled-components/macro'; +import React from 'react'; +import { withI18n } from '@lingui/react'; +import { t } from '@lingui/macro'; +import { + UrlFormField, + ScmCredentialFormField, + ScmTypeOptions, +} from './SharedFields'; + +const ArchiveSubForm = ({ + i18n, + credential, + onCredentialSelection, + scmUpdateOnLaunch, +}) => ( + <> + + {i18n._(t`Example URLs for Remote Archive Source Control include:`)} +
    +
  • https://github.com/username/project/archive/v0.0.1.tar.gz
    • +
  • https://github.com/username/project/archive/v0.0.2.zip
    • +
+ + } + /> + + + +); + +export default withI18n()(ArchiveSubForm); diff --git a/awx/ui_next/src/screens/Project/shared/ProjectSubForms/index.js b/awx/ui_next/src/screens/Project/shared/ProjectSubForms/index.js index 022673187f..8cf3e5594b 100644 --- a/awx/ui_next/src/screens/Project/shared/ProjectSubForms/index.js +++ b/awx/ui_next/src/screens/Project/shared/ProjectSubForms/index.js @@ -3,3 +3,4 @@ export { default as HgSubForm } from './HgSubForm'; export { default as InsightsSubForm } from './InsightsSubForm'; export { default as ManualSubForm } from './ManualSubForm'; export { default as SvnSubForm } from './SvnSubForm'; +export { default as ArchiveSubForm } from './ArchiveSubForm'; diff --git a/awx/ui_next/src/screens/Template/WorkflowJobTemplateVisualizer/Modals/NodeModals/NodeTypeStep/ProjectsList.jsx b/awx/ui_next/src/screens/Template/WorkflowJobTemplateVisualizer/Modals/NodeModals/NodeTypeStep/ProjectsList.jsx index e7dfa098c9..4e73c5b6e0 100644 --- a/awx/ui_next/src/screens/Template/WorkflowJobTemplateVisualizer/Modals/NodeModals/NodeTypeStep/ProjectsList.jsx +++ b/awx/ui_next/src/screens/Template/WorkflowJobTemplateVisualizer/Modals/NodeModals/NodeTypeStep/ProjectsList.jsx @@ -79,6 +79,7 @@ function ProjectsList({ i18n, nodeResource, onUpdateNodeResource }) { [`git`, i18n._(t`Git`)], [`hg`, i18n._(t`Mercurial`)], [`svn`, i18n._(t`Subversion`)], + [`archive`, i18n._(t`Remote Archive`)], [`insights`, i18n._(t`Red Hat Insights`)], ], }, diff --git a/awx/ui_next/src/types.js b/awx/ui_next/src/types.js index 7a66ae3c68..5ee55ef5d9 100644 --- a/awx/ui_next/src/types.js +++ b/awx/ui_next/src/types.js @@ -148,7 +148,7 @@ export const Project = shape({ created: string, name: string.isRequired, description: string, - scm_type: oneOf(['', 'git', 'hg', 'svn', 'insights']), + scm_type: oneOf(['', 'git', 'hg', 'svn', 'archive', 'insights']), scm_url: string, scm_branch: string, scm_refspec: string, From 2d23748971c155707f19e84b18e9ef4ff2a3c2a4 Mon Sep 17 00:00:00 2001 From: Philip Douglass Date: Fri, 21 Aug 2020 12:39:28 -0400 Subject: [PATCH 12/14] Add Remote Archive SCM Type feature to CHANGELOG.md Signed-off-by: Philip Douglass --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 60f79d4f57..3d6a657365 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ This is a list of high-level changes for each release of AWX. A full list of com ## 14.1.0 (TBD) - AWX images can now be built on ARM64 - https://github.com/ansible/awx/pull/7607 +- Added the Remote Archive SCM Type to support using immutable artifacts and releases (such as tarballs and zip files) as projects - https://github.com/ansible/awx/issues/7954 - Deprecated official support for Mercurial-based project updates - https://github.com/ansible/awx/issues/7932 - Added resource import/export support to the official AWX collection - https://github.com/ansible/awx/issues/7329 - Added the ability to import YAML-based resources (instead of just JSON) when using the AWX CLI - https://github.com/ansible/awx/pull/7808 From 30ae0f53ec083d05bc82e536688122909ea4adf6 Mon Sep 17 00:00:00 2001 From: Philip Douglass Date: Fri, 21 Aug 2020 13:09:09 -0400 Subject: [PATCH 13/14] Reject setting scm_branch for Remote Archive projects --- awx/api/serializers.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/awx/api/serializers.py b/awx/api/serializers.py index be6a9d640b..429bf512be 100644 --- a/awx/api/serializers.py +++ b/awx/api/serializers.py @@ -1336,6 +1336,8 @@ class ProjectOptionsSerializer(BaseSerializer): attrs.pop('local_path', None) if 'local_path' in attrs and attrs['local_path'] not in valid_local_paths: errors['local_path'] = _('This path is already being used by another manual project.') + if attrs.get('scm_branch') and scm_type == 'archive': + errors['scm_branch'] = _('SCM branch cannot be used with archive projects.') if attrs.get('scm_refspec') and scm_type != 'git': errors['scm_refspec'] = _('SCM refspec can only be used with git projects.') From 6dc41f54fc659d373c462b72bac25dc40c19be2f Mon Sep 17 00:00:00 2001 From: Philip Douglass Date: Sat, 22 Aug 2020 11:16:04 -0400 Subject: [PATCH 14/14] Ensure scm credentials are passed for archive scm type --- awx/main/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/awx/main/tasks.py b/awx/main/tasks.py index 06c740c129..3e70f93c19 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -2105,7 +2105,7 @@ class RunProjectUpdate(BaseTask): scm_username = False elif scm_url_parts.scheme.endswith('ssh'): scm_password = False - elif scm_type == 'insights': + elif scm_type in ('insights', 'archive'): extra_vars['scm_username'] = scm_username extra_vars['scm_password'] = scm_password scm_url = update_scm_url(scm_type, scm_url, scm_username,