From a952ab0a75dc2432575ceadea581b95810a57846 Mon Sep 17 00:00:00 2001 From: Seth Foster Date: Thu, 2 Mar 2023 19:29:00 -0500 Subject: [PATCH 1/5] Add scm_branch to inventory source and inventory update add scm_branch as optional field awxkit --- awx/api/serializers.py | 7 +++- .../0175_inventorysource_scm_branch.py | 32 +++++++++++++++++++ awx/main/models/inventory.py | 6 ++++ awx/main/tasks/jobs.py | 11 +++++-- .../plugins/modules/inventory_source.py | 7 ++++ .../test/awx/test_inventory_source.py | 1 + awxkit/awxkit/api/pages/inventory.py | 1 + 7 files changed, 61 insertions(+), 4 deletions(-) create mode 100644 awx/main/migrations/0175_inventorysource_scm_branch.py diff --git a/awx/api/serializers.py b/awx/api/serializers.py index 6e23f3298f..054180b51e 100644 --- a/awx/api/serializers.py +++ b/awx/api/serializers.py @@ -2000,6 +2000,7 @@ class InventorySourceOptionsSerializer(BaseSerializer): 'source', 'source_path', 'source_vars', + 'scm_branch', 'credential', 'enabled_var', 'enabled_value', @@ -2164,10 +2165,14 @@ class InventorySourceSerializer(UnifiedJobTemplateSerializer, InventorySourceOpt if ('source' in attrs or 'source_project' in attrs) and get_field_from_model_or_attrs('source_project') is None: raise serializers.ValidationError({"source_project": _("Project required for scm type sources.")}) else: - redundant_scm_fields = list(filter(lambda x: attrs.get(x, None), ['source_project', 'source_path'])) + redundant_scm_fields = list(filter(lambda x: attrs.get(x, None), ['source_project', 'source_path', 'scm_branch'])) if redundant_scm_fields: raise serializers.ValidationError({"detail": _("Cannot set %s if not SCM type." % ' '.join(redundant_scm_fields))}) + project = get_field_from_model_or_attrs('source_project') + if get_field_from_model_or_attrs('scm_branch') and not project.allow_override: + raise serializers.ValidationError({'scm_branch': _('Project does not allow overriding branch.')}) + attrs = super(InventorySourceSerializer, self).validate(attrs) # Check type consistency of source and cloud credential, if provided diff --git a/awx/main/migrations/0175_inventorysource_scm_branch.py b/awx/main/migrations/0175_inventorysource_scm_branch.py new file mode 100644 index 0000000000..56eed2da7f --- /dev/null +++ b/awx/main/migrations/0175_inventorysource_scm_branch.py @@ -0,0 +1,32 @@ +# Generated by Django 3.2.16 on 2023-03-03 20:44 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ('main', '0174_ensure_org_ee_admin_roles'), + ] + + operations = [ + migrations.AddField( + model_name='inventorysource', + name='scm_branch', + field=models.CharField( + blank=True, + default='', + help_text='Inventory source SCM branch. Project default used if blank. Only allowed if project allow_override field is set to true.', + max_length=1024, + ), + ), + migrations.AddField( + model_name='inventoryupdate', + name='scm_branch', + field=models.CharField( + blank=True, + default='', + help_text='Inventory source SCM branch. Project default used if blank. Only allowed if project allow_override field is set to true.', + max_length=1024, + ), + ), + ] diff --git a/awx/main/models/inventory.py b/awx/main/models/inventory.py index a0df4dfde4..7b55c51851 100644 --- a/awx/main/models/inventory.py +++ b/awx/main/models/inventory.py @@ -872,6 +872,12 @@ class InventorySourceOptions(BaseModel): default='', help_text=_('Inventory source variables in YAML or JSON format.'), ) + scm_branch = models.CharField( + max_length=1024, + default='', + blank=True, + help_text=_('Inventory source SCM branch. Project default used if blank. Only allowed if project allow_override field is set to true.'), + ) enabled_var = models.TextField( blank=True, default='', diff --git a/awx/main/tasks/jobs.py b/awx/main/tasks/jobs.py index 369e045e6f..9da4bc074b 100644 --- a/awx/main/tasks/jobs.py +++ b/awx/main/tasks/jobs.py @@ -759,7 +759,7 @@ class SourceControlMixin(BaseTask): def sync_and_copy(self, project, private_data_dir, scm_branch=None): self.acquire_lock(project, self.instance.id) - + is_commit = False try: original_branch = None failed_reason = project.get_reason_if_failed() @@ -771,6 +771,7 @@ class SourceControlMixin(BaseTask): if os.path.exists(project_path): git_repo = git.Repo(project_path) if git_repo.head.is_detached: + is_commit = True original_branch = git_repo.head.commit else: original_branch = git_repo.active_branch @@ -782,7 +783,11 @@ class SourceControlMixin(BaseTask): # for git project syncs, non-default branches can be problems # restore to branch the repo was on before this run try: - original_branch.checkout() + if is_commit: + git_repo.head.set_commit(original_branch) + git_repo.head.reset(index=True, working_tree=True) + else: + original_branch.checkout() except Exception: # this could have failed due to dirty tree, but difficult to predict all cases logger.exception(f'Failed to restore project repo to prior state after {self.instance.id}') @@ -1581,7 +1586,7 @@ class RunInventoryUpdate(SourceControlMixin, BaseTask): if inventory_update.source == 'scm': if not source_project: raise RuntimeError('Could not find project to run SCM inventory update from.') - self.sync_and_copy(source_project, private_data_dir) + self.sync_and_copy(source_project, private_data_dir, scm_branch=inventory_update.inventory_source.scm_branch) else: # If source is not SCM make an empty project directory, content is built inside inventory folder super(RunInventoryUpdate, self).build_project_dir(inventory_update, private_data_dir) diff --git a/awx_collection/plugins/modules/inventory_source.py b/awx_collection/plugins/modules/inventory_source.py index f3000ee9ec..1e6939df3f 100644 --- a/awx_collection/plugins/modules/inventory_source.py +++ b/awx_collection/plugins/modules/inventory_source.py @@ -105,6 +105,11 @@ options: description: - Project to use as source with scm option type: str + scm_branch: + description: + - Inventory source SCM branch. + - Project must have branch override enabled. + type: str state: description: - Desired state of the resource. @@ -178,6 +183,7 @@ def main(): update_on_launch=dict(type='bool'), update_cache_timeout=dict(type='int'), source_project=dict(), + scm_branch=dict(type='str'), notification_templates_started=dict(type="list", elements='str'), notification_templates_success=dict(type="list", elements='str'), notification_templates_error=dict(type="list", elements='str'), @@ -272,6 +278,7 @@ def main(): 'enabled_var', 'enabled_value', 'host_filter', + 'scm_branch', ) # Layer in all remaining optional information diff --git a/awx_collection/test/awx/test_inventory_source.py b/awx_collection/test/awx/test_inventory_source.py index bebd3fc00b..c3123ae9e5 100644 --- a/awx_collection/test/awx/test_inventory_source.py +++ b/awx_collection/test/awx/test_inventory_source.py @@ -112,6 +112,7 @@ def test_falsy_value(run_module, admin_user, base_inventory): # credential ? ? o o r r r r r r r o # source_project ? ? r - - - - - - - - - # source_path ? ? r - - - - - - - - - +# scm_branch ? ? r - - - - - - - - - # verbosity ? ? o o o o o o o o o o # overwrite ? ? o o o o o o o o o o # overwrite_vars ? ? o o o o o o o o o o diff --git a/awxkit/awxkit/api/pages/inventory.py b/awxkit/awxkit/api/pages/inventory.py index 1ff9e02960..eeace96bd6 100644 --- a/awxkit/awxkit/api/pages/inventory.py +++ b/awxkit/awxkit/api/pages/inventory.py @@ -319,6 +319,7 @@ class InventorySource(HasCreate, HasNotifications, UnifiedJobTemplate): optional_fields = ( 'source_path', 'source_vars', + 'scm_branch', 'timeout', 'overwrite', 'overwrite_vars', From 8e6f4fae8045f053336ae93fb77542d710451f1a Mon Sep 17 00:00:00 2001 From: Gabe Muniz Date: Fri, 3 Mar 2023 01:22:12 -0500 Subject: [PATCH 2/5] enable scm branch ui work --- .../InventorySourceDetail/InventorySourceDetail.js | 6 ++++++ .../screens/Inventory/shared/Inventory.helptext.js | 1 + .../screens/Inventory/shared/InventorySourceForm.js | 2 ++ .../shared/InventorySourceSubForms/SCMSubForm.js | 6 ++---- .../shared/InventorySourceSubForms/SharedFields.js | 13 +++++++++++++ 5 files changed, 24 insertions(+), 4 deletions(-) diff --git a/awx/ui/src/screens/Inventory/InventorySourceDetail/InventorySourceDetail.js b/awx/ui/src/screens/Inventory/InventorySourceDetail/InventorySourceDetail.js index 87e9ece5cd..513a6fc473 100644 --- a/awx/ui/src/screens/Inventory/InventorySourceDetail/InventorySourceDetail.js +++ b/awx/ui/src/screens/Inventory/InventorySourceDetail/InventorySourceDetail.js @@ -48,6 +48,7 @@ function InventorySourceDetail({ inventorySource }) { source, source_path, source_vars, + scm_branch, update_cache_timeout, update_on_launch, verbosity, @@ -233,6 +234,11 @@ function InventorySourceDetail({ inventorySource }) { helpText={helpText.subFormVerbosityFields} value={VERBOSITY()[verbosity]} /> + ({ }, enabledVariableField: t`Retrieve the enabled state from the given dict of host variables. The enabled variable may be specified using dot notation, e.g: 'foo.bar'`, + sourceControlBranch: t`Branch to use on inventory sync. Project default used if blank. Only allowed if project allow_override field is set to true.`, enabledValue: t`This field is ignored unless an Enabled Variable is set. If the enabled variable matches this value, the host will be enabled on import.`, hostFilter: t`Regular expression where only matching host names will be imported. The filter is applied as a post-processing step after any inventory plugin filters are applied.`, sourceVars: (docsBaseUrl, source) => { diff --git a/awx/ui/src/screens/Inventory/shared/InventorySourceForm.js b/awx/ui/src/screens/Inventory/shared/InventorySourceForm.js index 7131ed16bf..88baa8d03d 100644 --- a/awx/ui/src/screens/Inventory/shared/InventorySourceForm.js +++ b/awx/ui/src/screens/Inventory/shared/InventorySourceForm.js @@ -71,6 +71,7 @@ const InventorySourceFormFields = ({ source_project: null, source_script: null, source_vars: '---\n', + scm_branch: null, update_cache_timeout: 0, update_on_launch: false, verbosity: 1, @@ -248,6 +249,7 @@ const InventorySourceForm = ({ source_project: source?.summary_fields?.source_project || null, source_script: source?.summary_fields?.source_script || null, source_vars: source?.source_vars || '---\n', + scm_branch: source?.scm_branch || '', update_cache_timeout: source?.update_cache_timeout || 0, update_on_launch: source?.update_on_launch || false, verbosity: source?.verbosity || 1, diff --git a/awx/ui/src/screens/Inventory/shared/InventorySourceSubForms/SCMSubForm.js b/awx/ui/src/screens/Inventory/shared/InventorySourceSubForms/SCMSubForm.js index 1b6d3e95ba..873275c61e 100644 --- a/awx/ui/src/screens/Inventory/shared/InventorySourceSubForms/SCMSubForm.js +++ b/awx/ui/src/screens/Inventory/shared/InventorySourceSubForms/SCMSubForm.js @@ -20,6 +20,7 @@ import { EnabledVarField, EnabledValueField, HostFilterField, + BranchFormField, } from './SharedFields'; import getHelpText from '../Inventory.helptext'; @@ -36,7 +37,6 @@ const SCMSubForm = ({ autoPopulateProject }) => { name: 'source_path', validate: required(t`Select a value for this field`), }); - const { error: sourcePathError, request: fetchSourcePath } = useRequest( useCallback(async (projectId) => { const { data } = await ProjectsAPI.readInventories(projectId); @@ -44,7 +44,6 @@ const SCMSubForm = ({ autoPopulateProject }) => { }, []), [] ); - useEffect(() => { if (projectMeta.initialValue) { fetchSourcePath(projectMeta.initialValue.id); @@ -68,7 +67,6 @@ const SCMSubForm = ({ autoPopulateProject }) => { }, [fetchSourcePath, setFieldValue, setFieldTouched, sourcePathField.value] ); - const handleCredentialUpdate = useCallback( (value) => { setFieldValue('credential', value); @@ -76,9 +74,9 @@ const SCMSubForm = ({ autoPopulateProject }) => { }, [setFieldValue, setFieldTouched] ); - return ( <> + { /> ); }; + +export const BranchFormField = ({ label }) => { + const helpText = getHelpText(); + return ( + + ); +}; From a5f9506f49c978812ecbb178d2cc66337e5d9cf5 Mon Sep 17 00:00:00 2001 From: Seth Foster Date: Mon, 6 Mar 2023 23:41:34 -0500 Subject: [PATCH 3/5] spelling add allow_override to source_project --- awx/api/serializers.py | 2 +- .../Inventory/InventorySourceDetail/InventorySourceDetail.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/awx/api/serializers.py b/awx/api/serializers.py index 054180b51e..892c7dd68a 100644 --- a/awx/api/serializers.py +++ b/awx/api/serializers.py @@ -156,7 +156,7 @@ SUMMARIZABLE_FK_FIELDS = { 'default_environment': DEFAULT_SUMMARY_FIELDS + ('image',), 'execution_environment': DEFAULT_SUMMARY_FIELDS + ('image',), 'project': DEFAULT_SUMMARY_FIELDS + ('status', 'scm_type', 'allow_override'), - 'source_project': DEFAULT_SUMMARY_FIELDS + ('status', 'scm_type'), + 'source_project': DEFAULT_SUMMARY_FIELDS + ('status', 'scm_type', 'allow_override'), 'project_update': DEFAULT_SUMMARY_FIELDS + ('status', 'failed'), 'credential': DEFAULT_SUMMARY_FIELDS + ('kind', 'cloud', 'kubernetes', 'credential_type_id'), 'signature_validation_credential': DEFAULT_SUMMARY_FIELDS + ('kind', 'credential_type_id'), diff --git a/awx/ui/src/screens/Inventory/InventorySourceDetail/InventorySourceDetail.js b/awx/ui/src/screens/Inventory/InventorySourceDetail/InventorySourceDetail.js index 513a6fc473..cf5d1354af 100644 --- a/awx/ui/src/screens/Inventory/InventorySourceDetail/InventorySourceDetail.js +++ b/awx/ui/src/screens/Inventory/InventorySourceDetail/InventorySourceDetail.js @@ -235,7 +235,7 @@ function InventorySourceDetail({ inventorySource }) { value={VERBOSITY()[verbosity]} /> From a7b4c03188c31810543ff4ba560b8b8b63e12b11 Mon Sep 17 00:00:00 2001 From: Vidya Nambiar Date: Tue, 7 Mar 2023 14:57:07 -0500 Subject: [PATCH 4/5] Show scm_branch if project allows branch override --- .../shared/InventorySourceSubForms/SCMSubForm.js | 13 +++++++++++-- .../shared/InventorySourceSubForms/SharedFields.js | 13 ------------- 2 files changed, 11 insertions(+), 15 deletions(-) diff --git a/awx/ui/src/screens/Inventory/shared/InventorySourceSubForms/SCMSubForm.js b/awx/ui/src/screens/Inventory/shared/InventorySourceSubForms/SCMSubForm.js index 873275c61e..1217ca3ba0 100644 --- a/awx/ui/src/screens/Inventory/shared/InventorySourceSubForms/SCMSubForm.js +++ b/awx/ui/src/screens/Inventory/shared/InventorySourceSubForms/SCMSubForm.js @@ -20,9 +20,9 @@ import { EnabledVarField, EnabledValueField, HostFilterField, - BranchFormField, } from './SharedFields'; import getHelpText from '../Inventory.helptext'; +import FormField from 'components/FormField'; const SCMSubForm = ({ autoPopulateProject }) => { const helpText = getHelpText(); @@ -57,6 +57,7 @@ const SCMSubForm = ({ autoPopulateProject }) => { (value) => { setFieldValue('source_project', value); setFieldTouched('source_project', true, false); + setFieldValue('scm_branch', '', false); if (sourcePathField.value) { setFieldValue('source_path', ''); setFieldTouched('source_path', false); @@ -76,7 +77,15 @@ const SCMSubForm = ({ autoPopulateProject }) => { ); return ( <> - + {projectField.value?.allow_override && ( + + )} { /> ); }; - -export const BranchFormField = ({ label }) => { - const helpText = getHelpText(); - return ( - - ); -}; From ab6511a833ef1afd08b47060f5d10a2bd4cb0759 Mon Sep 17 00:00:00 2001 From: Seth Foster Date: Tue, 7 Mar 2023 15:38:03 -0500 Subject: [PATCH 5/5] fix ui lint --- .../Inventory/shared/InventorySourceSubForms/SCMSubForm.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/awx/ui/src/screens/Inventory/shared/InventorySourceSubForms/SCMSubForm.js b/awx/ui/src/screens/Inventory/shared/InventorySourceSubForms/SCMSubForm.js index 1217ca3ba0..043ddcebe7 100644 --- a/awx/ui/src/screens/Inventory/shared/InventorySourceSubForms/SCMSubForm.js +++ b/awx/ui/src/screens/Inventory/shared/InventorySourceSubForms/SCMSubForm.js @@ -13,6 +13,7 @@ import { required } from 'util/validators'; import CredentialLookup from 'components/Lookup/CredentialLookup'; import ProjectLookup from 'components/Lookup/ProjectLookup'; import Popover from 'components/Popover'; +import FormField from 'components/FormField'; import { OptionsField, SourceVarsField, @@ -22,7 +23,6 @@ import { HostFilterField, } from './SharedFields'; import getHelpText from '../Inventory.helptext'; -import FormField from 'components/FormField'; const SCMSubForm = ({ autoPopulateProject }) => { const helpText = getHelpText();