From b1ce5e24e3c5499440166a567766e6f86f64cdaf Mon Sep 17 00:00:00 2001 From: "Keith J. Grant" Date: Mon, 22 Feb 2021 14:48:48 -0800 Subject: [PATCH 01/51] add expand button to VariablesField --- awx/ui_next/.eslintrc | 3 +- .../src/components/CodeEditor/CodeEditor.jsx | 14 ++- .../components/CodeEditor/VariablesDetail.jsx | 89 +++++++++----- .../components/CodeEditor/VariablesField.jsx | 113 +++++++++++++++--- .../src/components/DetailList/CodeDetail.jsx | 13 +- .../src/screens/Host/HostFacts/HostFacts.jsx | 2 +- .../InventoryHostFacts/InventoryHostFacts.jsx | 2 +- .../screens/Setting/shared/SharedFields.jsx | 2 +- 8 files changed, 172 insertions(+), 66 deletions(-) diff --git a/awx/ui_next/.eslintrc b/awx/ui_next/.eslintrc index f82900134e..b7c86c305a 100644 --- a/awx/ui_next/.eslintrc +++ b/awx/ui_next/.eslintrc @@ -77,7 +77,8 @@ "resizeOrientation", "src", "theme", - "gridColumns" + "gridColumns", + "rows" ], "ignore": ["Ansible", "Tower", "JSON", "YAML", "lg"], "ignoreComponent": [ diff --git a/awx/ui_next/src/components/CodeEditor/CodeEditor.jsx b/awx/ui_next/src/components/CodeEditor/CodeEditor.jsx index 0218a83c3c..981cf01610 100644 --- a/awx/ui_next/src/components/CodeEditor/CodeEditor.jsx +++ b/awx/ui_next/src/components/CodeEditor/CodeEditor.jsx @@ -77,6 +77,13 @@ function CodeEditor({ className, i18n, }) { + if (typeof rows !== 'number' && rows !== 'auto') { + // eslint-disable-next-line no-console + console.warning( + `CodeEditor: Unexpected value for 'rows': ${rows}; expected number or 'auto'` + ); + } + const wrapper = useRef(null); const editor = useRef(null); @@ -117,7 +124,8 @@ function CodeEditor({ jinja2: 'django', }; - const numRows = fullHeight ? value.split('\n').length : rows; + const numRows = rows === 'auto' ? value.split('\n').length : rows; + const height = fullHeight ? '50vh' : `${numRows * LINE_HEIGHT + PADDING}px`; return ( <> @@ -132,7 +140,7 @@ function CodeEditor({ editorProps={{ $blockScrolling: true }} fontSize={16} width="100%" - height={`${numRows * LINE_HEIGHT + PADDING}px`} + height={height} hasErrors={hasErrors} setOptions={{ readOnly, @@ -178,7 +186,7 @@ CodeEditor.propTypes = { readOnly: bool, hasErrors: bool, fullHeight: bool, - rows: number, + rows: oneOf(number, string), className: string, }; CodeEditor.defaultProps = { diff --git a/awx/ui_next/src/components/CodeEditor/VariablesDetail.jsx b/awx/ui_next/src/components/CodeEditor/VariablesDetail.jsx index f5cfd91373..602ef6695f 100644 --- a/awx/ui_next/src/components/CodeEditor/VariablesDetail.jsx +++ b/awx/ui_next/src/components/CodeEditor/VariablesDetail.jsx @@ -2,7 +2,14 @@ import 'styled-components/macro'; import React, { useState, useEffect } from 'react'; import { node, number, oneOfType, shape, string, arrayOf } from 'prop-types'; import { Trans, withI18n } from '@lingui/react'; -import { Split, SplitItem, TextListItemVariants } from '@patternfly/react-core'; +import { t } from '@lingui/macro'; +import { + Split, + SplitItem, + TextListItemVariants, + Button, +} from '@patternfly/react-core'; +import { ExpandArrowsAltIcon } from '@patternfly/react-icons'; import { DetailName, DetailValue } from '../DetailList'; import MultiButtonToggle from '../MultiButtonToggle'; import Popover from '../Popover'; @@ -29,13 +36,22 @@ function getValueAsMode(value, mode) { return mode === YAML_MODE ? jsonToYaml(value) : yamlToJson(value); } -function VariablesDetail({ dataCy, helpText, value, label, rows, fullHeight }) { +function VariablesDetail({ + dataCy, + helpText, + value, + label, + rows, + fullHeight, + i18n, +}) { const [mode, setMode] = useState( isJsonObject(value) || isJsonString(value) ? JSON_MODE : YAML_MODE ); const [currentValue, setCurrentValue] = useState( isJsonObject(value) ? JSON.stringify(value, null, 2) : value || '---' ); + const [isExpanded, setIsExpanded] = useState(false); const [error, setError] = useState(null); useEffect(() => { @@ -61,35 +77,48 @@ function VariablesDetail({ dataCy, helpText, value, label, rows, fullHeight }) { css="grid-column: 1 / -1" > - -
- - {label} - - {helpText && ( - - )} -
+ + + +
+ + {label} + + {helpText && ( + + )} +
+ + + { + try { + setCurrentValue(getValueAsMode(currentValue, newMode)); + setMode(newMode); + } catch (err) { + setError(err); + } + }} + /> + + - { - try { - setCurrentValue(getValueAsMode(currentValue, newMode)); - setMode(newMode); - } catch (err) { - setError(err); - } - }} - /> + @@ -122,7 +151,7 @@ function VariablesDetail({ dataCy, helpText, value, label, rows, fullHeight }) { VariablesDetail.propTypes = { value: oneOfType([shape({}), arrayOf(string), string]).isRequired, label: node.isRequired, - rows: number, + rows: oneOfType(number, string), dataCy: string, helpText: string, }; diff --git a/awx/ui_next/src/components/CodeEditor/VariablesField.jsx b/awx/ui_next/src/components/CodeEditor/VariablesField.jsx index 8fbf1e4bfa..35756132e8 100644 --- a/awx/ui_next/src/components/CodeEditor/VariablesField.jsx +++ b/awx/ui_next/src/components/CodeEditor/VariablesField.jsx @@ -4,7 +4,8 @@ import { withI18n } from '@lingui/react'; import { t } from '@lingui/macro'; import { useField } from 'formik'; import styled from 'styled-components'; -import { Split, SplitItem } from '@patternfly/react-core'; +import { Split, SplitItem, Button, Modal } from '@patternfly/react-core'; +import { ExpandArrowsAltIcon } from '@patternfly/react-icons'; import { CheckboxField } from '../FormField'; import MultiButtonToggle from '../MultiButtonToggle'; import { yamlToJson, jsonToYaml, isJsonString } from '../../util/yaml'; @@ -20,6 +21,7 @@ const FieldHeader = styled.div` const StyledCheckboxField = styled(CheckboxField)` --pf-c-check__label--FontSize: var(--pf-c-form__label--FontSize); + margin-left: auto; `; function VariablesField({ @@ -31,10 +33,91 @@ function VariablesField({ promptId, tooltip, }) { - const [field, meta, helpers] = useField(name); + const [field, meta] = useField(name); const [mode, setMode] = useState( isJsonString(field.value) ? JSON_MODE : YAML_MODE ); + const [isExpanded, setIsExpanded] = useState(false); + + return ( + <> + setIsExpanded(true)} + mode={mode} + setMode={setMode} + /> + setIsExpanded(false)} + actions={[ + , + ]} + > +
+ +
+ + {meta.error ? ( +
+ {meta.error} +
+ ) : null} + + ); +} +VariablesField.propTypes = { + id: string.isRequired, + name: string.isRequired, + label: string.isRequired, + readOnly: bool, + promptId: string, +}; +VariablesField.defaultProps = { + readOnly: false, + promptId: null, +}; + +function VariablesFieldInternals({ + i18n, + id, + name, + label, + readOnly, + promptId, + tooltip, + fullHeight, + mode, + setMode, + onExpand, +}) { + const [field, meta, helpers] = useField(name); return (
@@ -75,6 +158,15 @@ function VariablesField({ name="ask_variables_on_launch" /> )} + {onExpand && ( + + )} { helpers.setValue(newVal); }} + fullHeight={fullHeight} hasErrors={!!meta.error} /> - {meta.error ? ( -
- {meta.error} -
- ) : null}
); } -VariablesField.propTypes = { - id: string.isRequired, - name: string.isRequired, - label: string.isRequired, - readOnly: bool, - promptId: string, -}; -VariablesField.defaultProps = { - readOnly: false, - promptId: null, -}; export default withI18n()(VariablesField); diff --git a/awx/ui_next/src/components/DetailList/CodeDetail.jsx b/awx/ui_next/src/components/DetailList/CodeDetail.jsx index 08c935b18e..90edb259e8 100644 --- a/awx/ui_next/src/components/DetailList/CodeDetail.jsx +++ b/awx/ui_next/src/components/DetailList/CodeDetail.jsx @@ -14,15 +14,7 @@ import { DetailName, DetailValue } from './Detail'; import CodeEditor from '../CodeEditor'; import Popover from '../Popover'; -function CodeDetail({ - value, - label, - mode, - rows, - fullHeight, - helpText, - dataCy, -}) { +function CodeDetail({ value, label, mode, rows, helpText, dataCy }) { const labelCy = dataCy ? `${dataCy}-label` : null; const valueCy = dataCy ? `${dataCy}-value` : null; @@ -57,7 +49,6 @@ function CodeDetail({ value={value} readOnly rows={rows} - fullHeight={fullHeight} css="margin-top: 10px" /> @@ -69,7 +60,7 @@ CodeDetail.propTypes = { label: node.isRequired, dataCy: string, helpText: string, - rows: number, + rows: oneOfType(number, string), mode: oneOf(['javascript', 'yaml', 'jinja2']).isRequired, }; CodeDetail.defaultProps = { diff --git a/awx/ui_next/src/screens/Host/HostFacts/HostFacts.jsx b/awx/ui_next/src/screens/Host/HostFacts/HostFacts.jsx index f33c989f5b..08b447a991 100644 --- a/awx/ui_next/src/screens/Host/HostFacts/HostFacts.jsx +++ b/awx/ui_next/src/screens/Host/HostFacts/HostFacts.jsx @@ -36,7 +36,7 @@ function HostFacts({ i18n, host }) { return ( - + ); diff --git a/awx/ui_next/src/screens/Inventory/InventoryHostFacts/InventoryHostFacts.jsx b/awx/ui_next/src/screens/Inventory/InventoryHostFacts/InventoryHostFacts.jsx index 6bffd37ba4..4d93ce58c3 100644 --- a/awx/ui_next/src/screens/Inventory/InventoryHostFacts/InventoryHostFacts.jsx +++ b/awx/ui_next/src/screens/Inventory/InventoryHostFacts/InventoryHostFacts.jsx @@ -35,7 +35,7 @@ function InventoryHostFacts({ i18n, host }) { return ( - + ); diff --git a/awx/ui_next/src/screens/Setting/shared/SharedFields.jsx b/awx/ui_next/src/screens/Setting/shared/SharedFields.jsx index b5e118088a..96dc973de5 100644 --- a/awx/ui_next/src/screens/Setting/shared/SharedFields.jsx +++ b/awx/ui_next/src/screens/Setting/shared/SharedFields.jsx @@ -286,7 +286,7 @@ const ObjectField = withI18n()(({ i18n, name, config, isRequired = false }) => { > { From d6a5a1e0d015d0156c07b8617a1f792177d15609 Mon Sep 17 00:00:00 2001 From: "Keith J. Grant" Date: Tue, 23 Feb 2021 16:33:11 -0800 Subject: [PATCH 02/51] add expand button to VariablesDetail --- .../components/CodeEditor/VariablesDetail.jsx | 174 ++++++++++++------ 1 file changed, 117 insertions(+), 57 deletions(-) diff --git a/awx/ui_next/src/components/CodeEditor/VariablesDetail.jsx b/awx/ui_next/src/components/CodeEditor/VariablesDetail.jsx index 602ef6695f..4c3af1fbe8 100644 --- a/awx/ui_next/src/components/CodeEditor/VariablesDetail.jsx +++ b/awx/ui_next/src/components/CodeEditor/VariablesDetail.jsx @@ -1,13 +1,14 @@ import 'styled-components/macro'; import React, { useState, useEffect } from 'react'; import { node, number, oneOfType, shape, string, arrayOf } from 'prop-types'; -import { Trans, withI18n } from '@lingui/react'; +import { withI18n } from '@lingui/react'; import { t } from '@lingui/macro'; import { Split, SplitItem, TextListItemVariants, Button, + Modal, } from '@patternfly/react-core'; import { ExpandArrowsAltIcon } from '@patternfly/react-icons'; import { DetailName, DetailValue } from '../DetailList'; @@ -36,15 +37,7 @@ function getValueAsMode(value, mode) { return mode === YAML_MODE ? jsonToYaml(value) : yamlToJson(value); } -function VariablesDetail({ - dataCy, - helpText, - value, - label, - rows, - fullHeight, - i18n, -}) { +function VariablesDetail({ dataCy, helpText, value, label, rows, i18n }) { const [mode, setMode] = useState( isJsonObject(value) || isJsonString(value) ? JSON_MODE : YAML_MODE ); @@ -76,51 +69,18 @@ function VariablesDetail({ fullWidth css="grid-column: 1 / -1" > - - - - -
- - {label} - - {helpText && ( - - )} -
- - - { - try { - setCurrentValue(getValueAsMode(currentValue, newMode)); - setMode(newMode); - } catch (err) { - setError(err); - } - }} - /> - - - - - - - + setIsExpanded(true)} + i18n={i18n} + /> {error && ( @@ -141,10 +100,48 @@ function VariablesDetail({ css="color: var(--pf-global--danger-color--100); font-size: var(--pf-global--FontSize--sm" > - Error: {error.message} + {i18n._(t`Error:`)} {error.message} )} + setIsExpanded(false)} + actions={[ + , + ]} + > +
+ + +
+ ); } @@ -161,4 +158,67 @@ VariablesDetail.defaultProps = { helpText: '', }; +function ModeToggle({ + label, + helpText, + dataCy, + currentValue, + setCurrentValue, + mode, + setMode, + setError, + onExpand, + i18n, +}) { + return ( + + + + +
+ + {label} + + {helpText && ( + + )} +
+ + + { + try { + setCurrentValue(getValueAsMode(currentValue, newMode)); + setMode(newMode); + } catch (err) { + setError(err); + } + }} + /> + + + + {onExpand && ( + + + + )} + + ); +} + export default withI18n()(VariablesDetail); From f867c9e4761441e2880b154b0d082f8cee3ec44b Mon Sep 17 00:00:00 2001 From: "Keith J. Grant" Date: Wed, 24 Feb 2021 08:54:36 -0800 Subject: [PATCH 03/51] fix type errors in VariablesDetail --- awx/ui_next/src/components/CodeEditor/CodeEditor.jsx | 6 +++--- awx/ui_next/src/components/CodeEditor/VariablesDetail.jsx | 2 +- .../src/components/CodeEditor/VariablesField.test.jsx | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/awx/ui_next/src/components/CodeEditor/CodeEditor.jsx b/awx/ui_next/src/components/CodeEditor/CodeEditor.jsx index 981cf01610..ad930b7a20 100644 --- a/awx/ui_next/src/components/CodeEditor/CodeEditor.jsx +++ b/awx/ui_next/src/components/CodeEditor/CodeEditor.jsx @@ -1,5 +1,5 @@ import React, { useEffect, useRef, useCallback } from 'react'; -import { oneOf, bool, number, string, func } from 'prop-types'; +import { oneOf, bool, number, string, func, oneOfType } from 'prop-types'; import ReactAce from 'react-ace'; import 'ace-builds/src-noconflict/mode-json'; import 'ace-builds/src-noconflict/mode-javascript'; @@ -77,7 +77,7 @@ function CodeEditor({ className, i18n, }) { - if (typeof rows !== 'number' && rows !== 'auto') { + if (rows && typeof rows !== 'number' && rows !== 'auto') { // eslint-disable-next-line no-console console.warning( `CodeEditor: Unexpected value for 'rows': ${rows}; expected number or 'auto'` @@ -186,7 +186,7 @@ CodeEditor.propTypes = { readOnly: bool, hasErrors: bool, fullHeight: bool, - rows: oneOf(number, string), + rows: oneOfType([number, string]), className: string, }; CodeEditor.defaultProps = { diff --git a/awx/ui_next/src/components/CodeEditor/VariablesDetail.jsx b/awx/ui_next/src/components/CodeEditor/VariablesDetail.jsx index 4c3af1fbe8..fa46a60f40 100644 --- a/awx/ui_next/src/components/CodeEditor/VariablesDetail.jsx +++ b/awx/ui_next/src/components/CodeEditor/VariablesDetail.jsx @@ -148,7 +148,7 @@ function VariablesDetail({ dataCy, helpText, value, label, rows, i18n }) { VariablesDetail.propTypes = { value: oneOfType([shape({}), arrayOf(string), string]).isRequired, label: node.isRequired, - rows: oneOfType(number, string), + rows: oneOfType([number, string]), dataCy: string, helpText: string, }; diff --git a/awx/ui_next/src/components/CodeEditor/VariablesField.test.jsx b/awx/ui_next/src/components/CodeEditor/VariablesField.test.jsx index e07ff9d40b..37cdc8a53d 100644 --- a/awx/ui_next/src/components/CodeEditor/VariablesField.test.jsx +++ b/awx/ui_next/src/components/CodeEditor/VariablesField.test.jsx @@ -32,7 +32,7 @@ describe('VariablesField', () => { ); const buttons = wrapper.find('Button'); - expect(buttons).toHaveLength(2); + expect(buttons).toHaveLength(3); expect(buttons.at(0).prop('variant')).toEqual('primary'); expect(buttons.at(1).prop('variant')).toEqual('secondary'); await act(async () => { From 659f68f280ad1b288acabc0dae7238cc76e9945b Mon Sep 17 00:00:00 2001 From: "Keith J. Grant" Date: Thu, 25 Feb 2021 09:44:45 -0800 Subject: [PATCH 04/51] add VariablesField expand test --- .../CodeEditor/VariablesField.test.jsx | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/awx/ui_next/src/components/CodeEditor/VariablesField.test.jsx b/awx/ui_next/src/components/CodeEditor/VariablesField.test.jsx index 37cdc8a53d..24c896069e 100644 --- a/awx/ui_next/src/components/CodeEditor/VariablesField.test.jsx +++ b/awx/ui_next/src/components/CodeEditor/VariablesField.test.jsx @@ -136,4 +136,27 @@ describe('VariablesField', () => { expect(wrapper.find('CodeEditor').prop('mode')).toEqual('javascript'); }); + + it('should open modal when expanded', async () => { + const value = '---'; + const wrapper = mountWithContexts( + + {formik => ( +
+ + + + )} +
+ ); + expect(wrapper.find('Modal').prop('isOpen')).toEqual(false); + + wrapper.find('Button[variant="plain"]').invoke('onClick')(); + wrapper.update(); + + expect(wrapper.find('Modal').prop('isOpen')).toEqual(true); + expect(wrapper.find('Modal CodeEditor')).toHaveLength(1); + }); }); From e8886a552563a2195af77331c712be0295cd2ae4 Mon Sep 17 00:00:00 2001 From: "Keith J. Grant" Date: Mon, 15 Mar 2021 11:42:09 -0700 Subject: [PATCH 05/51] fix InventoryGroupDetails test --- .../InventoryGroupDetail/InventoryGroupDetail.test.jsx | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/awx/ui_next/src/screens/Inventory/InventoryGroupDetail/InventoryGroupDetail.test.jsx b/awx/ui_next/src/screens/Inventory/InventoryGroupDetail/InventoryGroupDetail.test.jsx index a4c584c121..dc7e47b646 100644 --- a/awx/ui_next/src/screens/Inventory/InventoryGroupDetail/InventoryGroupDetail.test.jsx +++ b/awx/ui_next/src/screens/Inventory/InventoryGroupDetail/InventoryGroupDetail.test.jsx @@ -72,11 +72,12 @@ describe('', () => { }); test('should open delete modal and then call api to delete the group', async () => { + expect(wrapper.find('Modal').length).toBe(1); // variables modal already mounted await act(async () => { wrapper.find('button[aria-label="Delete"]').simulate('click'); }); - await waitForElement(wrapper, 'Modal', el => el.length === 1); - expect(wrapper.find('Modal').length).toBe(1); + wrapper.update(); + expect(wrapper.find('Modal').length).toBe(2); await act(async () => { wrapper.find('Radio[id="radio-delete"]').invoke('onChange')(); }); From fb257d0add5e2c55cc900e3d43aa0ae4f9433a8f Mon Sep 17 00:00:00 2001 From: "Keith J. Grant" Date: Wed, 24 Mar 2021 11:45:19 -0700 Subject: [PATCH 06/51] add ouia ids to variables expand buttons --- awx/ui_next/src/components/CodeEditor/VariablesDetail.jsx | 2 ++ awx/ui_next/src/components/CodeEditor/VariablesField.jsx | 2 ++ .../screens/Template/JobTemplateDetail/JobTemplateDetail.jsx | 1 + 3 files changed, 5 insertions(+) diff --git a/awx/ui_next/src/components/CodeEditor/VariablesDetail.jsx b/awx/ui_next/src/components/CodeEditor/VariablesDetail.jsx index fa46a60f40..797c10673d 100644 --- a/awx/ui_next/src/components/CodeEditor/VariablesDetail.jsx +++ b/awx/ui_next/src/components/CodeEditor/VariablesDetail.jsx @@ -115,6 +115,7 @@ function VariablesDetail({ dataCy, helpText, value, label, rows, i18n }) { key="select" variant="primary" onClick={() => setIsExpanded(false)} + ouiaId={`${dataCy}-unexpand`} > {i18n._(t`Done`)} , @@ -212,6 +213,7 @@ function ModeToggle({ variant="plain" aria-label={i18n._(t`Expand input`)} onClick={onExpand} + ouiaId={`${dataCy}-expand`} > diff --git a/awx/ui_next/src/components/CodeEditor/VariablesField.jsx b/awx/ui_next/src/components/CodeEditor/VariablesField.jsx index 35756132e8..6f039c6895 100644 --- a/awx/ui_next/src/components/CodeEditor/VariablesField.jsx +++ b/awx/ui_next/src/components/CodeEditor/VariablesField.jsx @@ -64,6 +64,7 @@ function VariablesField({ key="select" variant="primary" onClick={() => setIsExpanded(false)} + ouiaId={`${id}-variables-unexpand`} > {i18n._(t`Done`)} , @@ -163,6 +164,7 @@ function VariablesFieldInternals({ variant="plain" aria-label={i18n._(t`Expand input`)} onClick={onExpand} + ouiaId={`${id}-variables-expand`} > diff --git a/awx/ui_next/src/screens/Template/JobTemplateDetail/JobTemplateDetail.jsx b/awx/ui_next/src/screens/Template/JobTemplateDetail/JobTemplateDetail.jsx index d61819adcc..1a5cf4a4b1 100644 --- a/awx/ui_next/src/screens/Template/JobTemplateDetail/JobTemplateDetail.jsx +++ b/awx/ui_next/src/screens/Template/JobTemplateDetail/JobTemplateDetail.jsx @@ -372,6 +372,7 @@ function JobTemplateDetail({ i18n, template }) { value={extra_vars} rows={4} label={i18n._(t`Variables`)} + dataCy={`jt-details-${template.id}`} /> From 9390452f02560e9959e45a7484bcfbb332a8bb3c Mon Sep 17 00:00:00 2001 From: Dennis Hoppe Date: Thu, 25 Mar 2021 12:51:05 +0100 Subject: [PATCH 07/51] Set a custom name for Docker volumes --- .../ansible/roles/sources/templates/docker-compose.yml.j2 | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tools/docker-compose/ansible/roles/sources/templates/docker-compose.yml.j2 b/tools/docker-compose/ansible/roles/sources/templates/docker-compose.yml.j2 index 9810e7a7cf..3b32df378e 100644 --- a/tools/docker-compose/ansible/roles/sources/templates/docker-compose.yml.j2 +++ b/tools/docker-compose/ansible/roles/sources/templates/docker-compose.yml.j2 @@ -87,8 +87,11 @@ services: - "awx_db:/var/lib/postgresql/data" volumes: awx_db: + name: tools_awx_db {% for i in range(cluster_node_count|int) -%} {% set container_postfix = loop.index %} receptor_{{ container_postfix }}: + name: tools_receptor_{{ container_postfix }} redis_socket_{{ container_postfix }}: + name: tools_redis_socket_{{ container_postfix }} {% endfor -%} From b2665c084e918dea46ece5e098ecf46472c64e44 Mon Sep 17 00:00:00 2001 From: beeankha Date: Wed, 10 Mar 2021 16:59:11 -0500 Subject: [PATCH 08/51] Loosen Collections v Tower version check --- awx_collection/plugins/module_utils/tower_api.py | 14 +++++++++++--- requirements/requirements.in | 1 + 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/awx_collection/plugins/module_utils/tower_api.py b/awx_collection/plugins/module_utils/tower_api.py index 1ef663fb26..de81b6a662 100644 --- a/awx_collection/plugins/module_utils/tower_api.py +++ b/awx_collection/plugins/module_utils/tower_api.py @@ -7,6 +7,7 @@ from ansible.module_utils.urls import Request, SSLValidationError, ConnectionErr from ansible.module_utils.six import PY2 from ansible.module_utils.six.moves.urllib.error import HTTPError from ansible.module_utils.six.moves.http_cookiejar import CookieJar +import semver import time from json import loads, dumps @@ -259,10 +260,17 @@ class TowerAPIModule(TowerModule): tower_type = response.info().getheader('X-API-Product-Name', None) tower_version = response.info().getheader('X-API-Product-Version', None) + semver_collection_version = semver.VersionInfo.parse(self._COLLECTION_VERSION) + semver_tower_version = semver.VersionInfo.parse(tower_version) + if self._COLLECTION_TYPE not in self.collection_to_version or self.collection_to_version[self._COLLECTION_TYPE] != tower_type: - self.warn("You are using the {0} version of this collection but connecting to {1}".format(self._COLLECTION_TYPE, tower_type)) - elif self._COLLECTION_VERSION != tower_version: - self.warn("You are running collection version {0} but connecting to tower version {1}".format(self._COLLECTION_VERSION, tower_version)) + self.warn("You are using the {0} version of this collection but connecting to {1}".format( + self._COLLECTION_TYPE, tower_type + )) + elif semver_collection_version.major != semver_tower_version.major: + self.warn("You are running collection version {0} but connecting to tower version {1}".format( + self._COLLECTION_VERSION, tower_version + )) self.version_checked = True response_body = '' diff --git a/requirements/requirements.in b/requirements/requirements.in index 1970b215fb..4f22262c86 100644 --- a/requirements/requirements.in +++ b/requirements/requirements.in @@ -43,6 +43,7 @@ python3-saml python-ldap>=3.3.1 # https://github.com/python-ldap/python-ldap/issues/270 pyyaml>=5.4.1 # minimum to fix https://github.com/yaml/pyyaml/issues/478 schedule==0.6.0 +semver==2.13.0 social-auth-core==3.3.1 # see UPGRADE BLOCKERs social-auth-app-django==3.1.0 # see UPGRADE BLOCKERs redis From e2b290ff99d52ae89072db69a15b36198385561b Mon Sep 17 00:00:00 2001 From: beeankha Date: Tue, 23 Mar 2021 12:17:59 -0400 Subject: [PATCH 09/51] Use distutils instead of semver, add/update unit tests --- awx_collection/plugins/module_utils/tower_api.py | 16 ++++++---------- awx_collection/test/awx/test_module_utils.py | 16 +++++++++++++++- requirements/requirements.in | 1 - 3 files changed, 21 insertions(+), 12 deletions(-) diff --git a/awx_collection/plugins/module_utils/tower_api.py b/awx_collection/plugins/module_utils/tower_api.py index de81b6a662..02122b6c25 100644 --- a/awx_collection/plugins/module_utils/tower_api.py +++ b/awx_collection/plugins/module_utils/tower_api.py @@ -7,7 +7,7 @@ from ansible.module_utils.urls import Request, SSLValidationError, ConnectionErr from ansible.module_utils.six import PY2 from ansible.module_utils.six.moves.urllib.error import HTTPError from ansible.module_utils.six.moves.http_cookiejar import CookieJar -import semver +from distutils.version import LooseVersion as Version import time from json import loads, dumps @@ -260,17 +260,13 @@ class TowerAPIModule(TowerModule): tower_type = response.info().getheader('X-API-Product-Name', None) tower_version = response.info().getheader('X-API-Product-Version', None) - semver_collection_version = semver.VersionInfo.parse(self._COLLECTION_VERSION) - semver_tower_version = semver.VersionInfo.parse(tower_version) + collection_major = Version(self._COLLECTION_VERSION).version[0] + tower_major = Version(tower_version).version[0] if self._COLLECTION_TYPE not in self.collection_to_version or self.collection_to_version[self._COLLECTION_TYPE] != tower_type: - self.warn("You are using the {0} version of this collection but connecting to {1}".format( - self._COLLECTION_TYPE, tower_type - )) - elif semver_collection_version.major != semver_tower_version.major: - self.warn("You are running collection version {0} but connecting to tower version {1}".format( - self._COLLECTION_VERSION, tower_version - )) + self.warn("You are using the {0} version of this collection but connecting to {1}".format(self._COLLECTION_TYPE, tower_type)) + elif collection_major != tower_major: + self.warn("You are running collection version {0} but connecting to tower version {1}".format(self._COLLECTION_VERSION, tower_version)) self.version_checked = True response_body = '' diff --git a/awx_collection/test/awx/test_module_utils.py b/awx_collection/test/awx/test_module_utils.py index 473bfe9457..61a3b6d4bd 100644 --- a/awx_collection/test/awx/test_module_utils.py +++ b/awx_collection/test/awx/test_module_utils.py @@ -32,6 +32,20 @@ def mock_ping_response(self, method, url, **kwargs): def test_version_warning(collection_import, silence_warning): + TowerAPIModule = collection_import('plugins.module_utils.tower_api').TowerAPIModule + cli_data = {'ANSIBLE_MODULE_ARGS': {}} + testargs = ['module_file2.py', json.dumps(cli_data)] + with mock.patch.object(sys, 'argv', testargs): + with mock.patch('ansible.module_utils.urls.Request.open', new=mock_ping_response): + my_module = TowerAPIModule(argument_spec=dict()) + my_module._COLLECTION_VERSION = "2.0.0" + my_module._COLLECTION_TYPE = "not-junk" + my_module.collection_to_version['not-junk'] = 'not-junk' + my_module.get_endpoint('ping') + silence_warning.assert_called_once_with('You are running collection version 2.0.0 but connecting to tower version 1.2.3') + + +def test_version_warning_strictness(collection_import, silence_warning): TowerAPIModule = collection_import('plugins.module_utils.tower_api').TowerAPIModule cli_data = {'ANSIBLE_MODULE_ARGS': {}} testargs = ['module_file2.py', json.dumps(cli_data)] @@ -42,7 +56,7 @@ def test_version_warning(collection_import, silence_warning): my_module._COLLECTION_TYPE = "not-junk" my_module.collection_to_version['not-junk'] = 'not-junk' my_module.get_endpoint('ping') - silence_warning.assert_called_once_with('You are running collection version 1.0.0 but connecting to tower version 1.2.3') + silence_warning.assert_not_called() def test_type_warning(collection_import, silence_warning): diff --git a/requirements/requirements.in b/requirements/requirements.in index 4f22262c86..1970b215fb 100644 --- a/requirements/requirements.in +++ b/requirements/requirements.in @@ -43,7 +43,6 @@ python3-saml python-ldap>=3.3.1 # https://github.com/python-ldap/python-ldap/issues/270 pyyaml>=5.4.1 # minimum to fix https://github.com/yaml/pyyaml/issues/478 schedule==0.6.0 -semver==2.13.0 social-auth-core==3.3.1 # see UPGRADE BLOCKERs social-auth-app-django==3.1.0 # see UPGRADE BLOCKERs redis From bb43ecb0b510cb2200d7459c98a80a46cd6c3205 Mon Sep 17 00:00:00 2001 From: John Westcott IV Date: Wed, 24 Mar 2021 10:16:53 -0400 Subject: [PATCH 10/51] Splitting out AWX and Tower versions --- .../plugins/module_utils/tower_api.py | 14 +++- awx_collection/test/awx/test_module_utils.py | 70 +++++++++++++++---- 2 files changed, 66 insertions(+), 18 deletions(-) diff --git a/awx_collection/plugins/module_utils/tower_api.py b/awx_collection/plugins/module_utils/tower_api.py index 02122b6c25..beea9868d2 100644 --- a/awx_collection/plugins/module_utils/tower_api.py +++ b/awx_collection/plugins/module_utils/tower_api.py @@ -260,13 +260,21 @@ class TowerAPIModule(TowerModule): tower_type = response.info().getheader('X-API-Product-Name', None) tower_version = response.info().getheader('X-API-Product-Version', None) - collection_major = Version(self._COLLECTION_VERSION).version[0] - tower_major = Version(tower_version).version[0] + parsed_collection_version = Version(self._COLLECTION_VERSION).version + parsed_tower_version = Version(tower_version).version + if tower_type != 'AWX': + collection_compare_ver = parsed_collection_version[0] + tower_compare_ver = parsed_tower_version[0] + else: + collection_compare_ver = "{}.{}".format(parsed_collection_version[0], parsed_collection_version[1]) + tower_major = '{}.{}'.format(parsed_tower_version[0], parsed_tower_version[1]) + if self._COLLECTION_TYPE not in self.collection_to_version or self.collection_to_version[self._COLLECTION_TYPE] != tower_type: self.warn("You are using the {0} version of this collection but connecting to {1}".format(self._COLLECTION_TYPE, tower_type)) - elif collection_major != tower_major: + elif collection_compare_ver != tower_compare_ver: self.warn("You are running collection version {0} but connecting to tower version {1}".format(self._COLLECTION_VERSION, tower_version)) + self.version_checked = True response_body = '' diff --git a/awx_collection/test/awx/test_module_utils.py b/awx_collection/test/awx/test_module_utils.py index 61a3b6d4bd..168c5c7f27 100644 --- a/awx_collection/test/awx/test_module_utils.py +++ b/awx_collection/test/awx/test_module_utils.py @@ -10,8 +10,12 @@ from requests.models import Response from unittest import mock -def getheader(self, header_name, default): - mock_headers = {'X-API-Product-Name': 'not-junk', 'X-API-Product-Version': '1.2.3'} +def getTowerheader(self, header_name, default): + mock_headers = {'X-API-Product-Name': 'Red Hat Ansible Tower', 'X-API-Product-Version': '1.2.3'} + return mock_headers.get(header_name, default) + +def getAWXheader(self, header_name, default): + mock_headers = {'X-API-Product-Name': 'AWX', 'X-API-Product-Version': '1.2.3'} return mock_headers.get(header_name, default) @@ -23,9 +27,16 @@ def status(self): return 200 -def mock_ping_response(self, method, url, **kwargs): +def mock_tower_ping_response(self, method, url, **kwargs): r = Response() - r.getheader = getheader.__get__(r) + r.getheader = getTowerheader.__get__(r) + r.read = read.__get__(r) + r.status = status.__get__(r) + return r + +def mock_awx_ping_response(self, method, url, **kwargs): + r = Response() + r.getheader = getAWXheader.__get__(r) r.read = read.__get__(r) r.status = status.__get__(r) return r @@ -36,41 +47,70 @@ def test_version_warning(collection_import, silence_warning): cli_data = {'ANSIBLE_MODULE_ARGS': {}} testargs = ['module_file2.py', json.dumps(cli_data)] with mock.patch.object(sys, 'argv', testargs): - with mock.patch('ansible.module_utils.urls.Request.open', new=mock_ping_response): + with mock.patch('ansible.module_utils.urls.Request.open', new=mock_awx_ping_response): my_module = TowerAPIModule(argument_spec=dict()) my_module._COLLECTION_VERSION = "2.0.0" - my_module._COLLECTION_TYPE = "not-junk" - my_module.collection_to_version['not-junk'] = 'not-junk' + my_module._COLLECTION_TYPE = "awx" my_module.get_endpoint('ping') silence_warning.assert_called_once_with('You are running collection version 2.0.0 but connecting to tower version 1.2.3') -def test_version_warning_strictness(collection_import, silence_warning): +def test_version_warning_strictness_awx(collection_import, silence_warning): TowerAPIModule = collection_import('plugins.module_utils.tower_api').TowerAPIModule cli_data = {'ANSIBLE_MODULE_ARGS': {}} testargs = ['module_file2.py', json.dumps(cli_data)] + # Compare 1.0.0 to 1.2.3 (major matches) with mock.patch.object(sys, 'argv', testargs): - with mock.patch('ansible.module_utils.urls.Request.open', new=mock_ping_response): + with mock.patch('ansible.module_utils.urls.Request.open', new=mock_awx_ping_response): my_module = TowerAPIModule(argument_spec=dict()) my_module._COLLECTION_VERSION = "1.0.0" - my_module._COLLECTION_TYPE = "not-junk" - my_module.collection_to_version['not-junk'] = 'not-junk' + my_module._COLLECTION_TYPE = "awx" my_module.get_endpoint('ping') silence_warning.assert_not_called() + # Compare 1.2.0 to 1.2.3 (major matches minor does not count) + with mock.patch.object(sys, 'argv', testargs): + with mock.patch('ansible.module_utils.urls.Request.open', new=mock_awx_ping_response): + my_module = TowerAPIModule(argument_spec=dict()) + my_module._COLLECTION_VERSION = "1.2.0" + my_module._COLLECTION_TYPE = "awx" + my_module.get_endpoint('ping') + silence_warning.assert_not_called() + +def test_version_warning_strictness_tower(collection_import, silence_warning): + TowerAPIModule = collection_import('plugins.module_utils.tower_api').TowerAPIModule + cli_data = {'ANSIBLE_MODULE_ARGS': {}} + testargs = ['module_file2.py', json.dumps(cli_data)] + # Compare 1.2.0 to 1.2.3 (major/minor matches) + with mock.patch.object(sys, 'argv', testargs): + with mock.patch('ansible.module_utils.urls.Request.open', new=mock_tower_ping_response): + my_module = TowerAPIModule(argument_spec=dict()) + my_module._COLLECTION_VERSION = "1.2.0" + my_module.get_endpoint('ping') + silence_warning.assert_not_called() + + # Compare 1.0.0 to 1.2.3 (major/minor fail to match) + with mock.patch.object(sys, 'argv', testargs): + with mock.patch('ansible.module_utils.urls.Request.open', new=mock_tower_ping_response): + my_module = TowerAPIModule(argument_spec=dict()) + my_module._COLLECTION_VERSION = "1.0.0" + my_module._COLLECTION_TYPE = "tower" + my_module.get_endpoint('ping') + silence_warning.assert_called_once_with('You are running collection version 1.0.0 but connecting to tower version 1.2.3') + + def test_type_warning(collection_import, silence_warning): TowerAPIModule = collection_import('plugins.module_utils.tower_api').TowerAPIModule cli_data = {'ANSIBLE_MODULE_ARGS': {}} testargs = ['module_file2.py', json.dumps(cli_data)] with mock.patch.object(sys, 'argv', testargs): - with mock.patch('ansible.module_utils.urls.Request.open', new=mock_ping_response): + with mock.patch('ansible.module_utils.urls.Request.open', new=mock_awx_ping_response): my_module = TowerAPIModule(argument_spec={}) my_module._COLLECTION_VERSION = "1.2.3" - my_module._COLLECTION_TYPE = "junk" - my_module.collection_to_version['junk'] = 'junk' + my_module._COLLECTION_TYPE = "tower" my_module.get_endpoint('ping') - silence_warning.assert_called_once_with('You are using the junk version of this collection but connecting to not-junk') + silence_warning.assert_called_once_with('You are using the tower version of this collection but connecting to awx') def test_duplicate_config(collection_import, silence_warning): From aa9906ebae7a5bf86cb65fa54731a772a1c58db2 Mon Sep 17 00:00:00 2001 From: John Westcott IV Date: Thu, 25 Mar 2021 08:58:26 -0400 Subject: [PATCH 11/51] Fixing issues --- awx_collection/plugins/module_utils/tower_api.py | 2 +- awx_collection/test/awx/test_module_utils.py | 16 ++++++++++------ 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/awx_collection/plugins/module_utils/tower_api.py b/awx_collection/plugins/module_utils/tower_api.py index beea9868d2..4fcdecfde1 100644 --- a/awx_collection/plugins/module_utils/tower_api.py +++ b/awx_collection/plugins/module_utils/tower_api.py @@ -267,7 +267,7 @@ class TowerAPIModule(TowerModule): tower_compare_ver = parsed_tower_version[0] else: collection_compare_ver = "{}.{}".format(parsed_collection_version[0], parsed_collection_version[1]) - tower_major = '{}.{}'.format(parsed_tower_version[0], parsed_tower_version[1]) + tower_compare_ver = '{}.{}'.format(parsed_tower_version[0], parsed_tower_version[1]) if self._COLLECTION_TYPE not in self.collection_to_version or self.collection_to_version[self._COLLECTION_TYPE] != tower_type: diff --git a/awx_collection/test/awx/test_module_utils.py b/awx_collection/test/awx/test_module_utils.py index 168c5c7f27..2c88e64826 100644 --- a/awx_collection/test/awx/test_module_utils.py +++ b/awx_collection/test/awx/test_module_utils.py @@ -9,13 +9,16 @@ from awx.main.models import Organization, Team, Project, Inventory from requests.models import Response from unittest import mock +awx_name = 'AWX' +tower_name = 'Red Hat Ansible Tower' +ping_version = '1.2.3' def getTowerheader(self, header_name, default): - mock_headers = {'X-API-Product-Name': 'Red Hat Ansible Tower', 'X-API-Product-Version': '1.2.3'} + mock_headers = {'X-API-Product-Name': tower_name, 'X-API-Product-Version': ping_version} return mock_headers.get(header_name, default) def getAWXheader(self, header_name, default): - mock_headers = {'X-API-Product-Name': 'AWX', 'X-API-Product-Version': '1.2.3'} + mock_headers = {'X-API-Product-Name': awx_name, 'X-API-Product-Version': ping_version} return mock_headers.get(header_name, default) @@ -52,7 +55,7 @@ def test_version_warning(collection_import, silence_warning): my_module._COLLECTION_VERSION = "2.0.0" my_module._COLLECTION_TYPE = "awx" my_module.get_endpoint('ping') - silence_warning.assert_called_once_with('You are running collection version 2.0.0 but connecting to tower version 1.2.3') + silence_warning.assert_called_once_with('You are running collection version {} but connecting to tower version {}'.format(my_module._COLLECTION_VERSION, ping_version)) def test_version_warning_strictness_awx(collection_import, silence_warning): @@ -86,6 +89,7 @@ def test_version_warning_strictness_tower(collection_import, silence_warning): with mock.patch('ansible.module_utils.urls.Request.open', new=mock_tower_ping_response): my_module = TowerAPIModule(argument_spec=dict()) my_module._COLLECTION_VERSION = "1.2.0" + my_module._COLLECTION_TYPE = "tower" my_module.get_endpoint('ping') silence_warning.assert_not_called() @@ -96,7 +100,7 @@ def test_version_warning_strictness_tower(collection_import, silence_warning): my_module._COLLECTION_VERSION = "1.0.0" my_module._COLLECTION_TYPE = "tower" my_module.get_endpoint('ping') - silence_warning.assert_called_once_with('You are running collection version 1.0.0 but connecting to tower version 1.2.3') + silence_warning.assert_called_once_with('You are running collection version {} but connecting to tower version {}'.format(my_module._COLLECTION_VERSION, ping_version)) @@ -107,10 +111,10 @@ def test_type_warning(collection_import, silence_warning): with mock.patch.object(sys, 'argv', testargs): with mock.patch('ansible.module_utils.urls.Request.open', new=mock_awx_ping_response): my_module = TowerAPIModule(argument_spec={}) - my_module._COLLECTION_VERSION = "1.2.3" + my_module._COLLECTION_VERSION = ping_version my_module._COLLECTION_TYPE = "tower" my_module.get_endpoint('ping') - silence_warning.assert_called_once_with('You are using the tower version of this collection but connecting to awx') + silence_warning.assert_called_once_with('You are using the {} version of this collection but connecting to {}'.format(my_module._COLLECTION_TYPE, awx_name)) def test_duplicate_config(collection_import, silence_warning): From 75a99bb1d50d30e39e107883822d7a4996b623c1 Mon Sep 17 00:00:00 2001 From: John Westcott IV Date: Thu, 25 Mar 2021 09:33:46 -0400 Subject: [PATCH 12/51] Fixing version check --- awx_collection/plugins/module_utils/tower_api.py | 3 +-- awx_collection/test/awx/test_module_utils.py | 13 ++++++++++--- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/awx_collection/plugins/module_utils/tower_api.py b/awx_collection/plugins/module_utils/tower_api.py index 4fcdecfde1..2edba2b502 100644 --- a/awx_collection/plugins/module_utils/tower_api.py +++ b/awx_collection/plugins/module_utils/tower_api.py @@ -262,14 +262,13 @@ class TowerAPIModule(TowerModule): parsed_collection_version = Version(self._COLLECTION_VERSION).version parsed_tower_version = Version(tower_version).version - if tower_type != 'AWX': + if tower_type == 'AWX': collection_compare_ver = parsed_collection_version[0] tower_compare_ver = parsed_tower_version[0] else: collection_compare_ver = "{}.{}".format(parsed_collection_version[0], parsed_collection_version[1]) tower_compare_ver = '{}.{}'.format(parsed_tower_version[0], parsed_tower_version[1]) - if self._COLLECTION_TYPE not in self.collection_to_version or self.collection_to_version[self._COLLECTION_TYPE] != tower_type: self.warn("You are using the {0} version of this collection but connecting to {1}".format(self._COLLECTION_TYPE, tower_type)) elif collection_compare_ver != tower_compare_ver: diff --git a/awx_collection/test/awx/test_module_utils.py b/awx_collection/test/awx/test_module_utils.py index 2c88e64826..8f2a8e52d5 100644 --- a/awx_collection/test/awx/test_module_utils.py +++ b/awx_collection/test/awx/test_module_utils.py @@ -13,10 +13,12 @@ awx_name = 'AWX' tower_name = 'Red Hat Ansible Tower' ping_version = '1.2.3' + def getTowerheader(self, header_name, default): mock_headers = {'X-API-Product-Name': tower_name, 'X-API-Product-Version': ping_version} return mock_headers.get(header_name, default) + def getAWXheader(self, header_name, default): mock_headers = {'X-API-Product-Name': awx_name, 'X-API-Product-Version': ping_version} return mock_headers.get(header_name, default) @@ -37,6 +39,7 @@ def mock_tower_ping_response(self, method, url, **kwargs): r.status = status.__get__(r) return r + def mock_awx_ping_response(self, method, url, **kwargs): r = Response() r.getheader = getAWXheader.__get__(r) @@ -55,7 +58,9 @@ def test_version_warning(collection_import, silence_warning): my_module._COLLECTION_VERSION = "2.0.0" my_module._COLLECTION_TYPE = "awx" my_module.get_endpoint('ping') - silence_warning.assert_called_once_with('You are running collection version {} but connecting to tower version {}'.format(my_module._COLLECTION_VERSION, ping_version)) + silence_warning.assert_called_once_with( + 'You are running collection version {} but connecting to tower version {}'.format(my_module._COLLECTION_VERSION, ping_version) + ) def test_version_warning_strictness_awx(collection_import, silence_warning): @@ -80,6 +85,7 @@ def test_version_warning_strictness_awx(collection_import, silence_warning): my_module.get_endpoint('ping') silence_warning.assert_not_called() + def test_version_warning_strictness_tower(collection_import, silence_warning): TowerAPIModule = collection_import('plugins.module_utils.tower_api').TowerAPIModule cli_data = {'ANSIBLE_MODULE_ARGS': {}} @@ -100,8 +106,9 @@ def test_version_warning_strictness_tower(collection_import, silence_warning): my_module._COLLECTION_VERSION = "1.0.0" my_module._COLLECTION_TYPE = "tower" my_module.get_endpoint('ping') - silence_warning.assert_called_once_with('You are running collection version {} but connecting to tower version {}'.format(my_module._COLLECTION_VERSION, ping_version)) - + silence_warning.assert_called_once_with( + 'You are running collection version {} but connecting to tower version {}'.format(my_module._COLLECTION_VERSION, ping_version) + ) def test_type_warning(collection_import, silence_warning): From db20bbe6820893bbe4979855c310de86afbbf255 Mon Sep 17 00:00:00 2001 From: Ryan Petrello Date: Wed, 24 Mar 2021 22:02:17 -0400 Subject: [PATCH 13/51] remove ansible_version from the API config and metrics endpoints AWX no longer includes Ansible on the control plane and there is no "default" version of Ansible aside from what's configured at the Execution Environment level see: https://github.com/ansible/awx/issues/9472 --- awx/api/views/root.py | 3 +-- awx/main/analytics/collectors.py | 5 ++--- awx/main/analytics/metrics.py | 3 +-- awx/main/utils/common.py | 15 --------------- awx/ui_next/src/components/About/About.jsx | 19 ++----------------- .../components/AppContainer/AppContainer.jsx | 1 - .../AppContainer/AppContainer.test.jsx | 3 --- 7 files changed, 6 insertions(+), 43 deletions(-) diff --git a/awx/api/views/root.py b/awx/api/views/root.py index 012d0c7c96..7a7ea649b1 100644 --- a/awx/api/views/root.py +++ b/awx/api/views/root.py @@ -24,7 +24,7 @@ from awx.api.generics import APIView from awx.conf.registry import settings_registry from awx.main.analytics import all_collectors from awx.main.ha import is_ha_environment -from awx.main.utils import get_awx_version, get_ansible_version, get_custom_venv_choices, to_python_boolean +from awx.main.utils import get_awx_version, get_custom_venv_choices, to_python_boolean from awx.main.utils.licensing import validate_entitlement_manifest from awx.api.versioning import reverse, drf_reverse from awx.main.constants import PRIVILEGE_ESCALATION_METHODS @@ -279,7 +279,6 @@ class ApiV2ConfigView(APIView): time_zone=settings.TIME_ZONE, license_info=license_data, version=get_awx_version(), - ansible_version=get_ansible_version(), eula=render_to_string("eula.md") if license_data.get('license_type', 'UNLICENSED') != 'open' else '', analytics_status=pendo_state, analytics_collectors=all_collectors(), diff --git a/awx/main/analytics/collectors.py b/awx/main/analytics/collectors.py index 3cb8ade69d..abdeb88b6c 100644 --- a/awx/main/analytics/collectors.py +++ b/awx/main/analytics/collectors.py @@ -11,7 +11,7 @@ from django.utils.timezone import now from django.utils.translation import ugettext_lazy as _ from awx.conf.license import get_license -from awx.main.utils import get_awx_version, get_ansible_version, get_custom_venv_choices, camelcase_to_underscore +from awx.main.utils import get_awx_version, get_custom_venv_choices, camelcase_to_underscore from awx.main import models from django.contrib.sessions.models import Session from awx.main.analytics import register @@ -33,7 +33,7 @@ data _since_ the last report date - i.e., new data in the last 24 hours) ''' -@register('config', '1.2', description=_('General platform configuration.')) +@register('config', '1.3', description=_('General platform configuration.')) def config(since, **kwargs): license_info = get_license() install_type = 'traditional' @@ -52,7 +52,6 @@ def config(since, **kwargs): 'instance_uuid': settings.SYSTEM_UUID, 'tower_url_base': settings.TOWER_URL_BASE, 'tower_version': get_awx_version(), - 'ansible_version': get_ansible_version(), 'license_type': license_info.get('license_type', 'UNLICENSED'), 'free_instances': license_info.get('free_instances', 0), 'total_licensed_instances': license_info.get('instance_count', 0), diff --git a/awx/main/analytics/metrics.py b/awx/main/analytics/metrics.py index e889719ded..3e5a244120 100644 --- a/awx/main/analytics/metrics.py +++ b/awx/main/analytics/metrics.py @@ -2,7 +2,7 @@ from django.conf import settings from prometheus_client import REGISTRY, PROCESS_COLLECTOR, PLATFORM_COLLECTOR, GC_COLLECTOR, Gauge, Info, generate_latest from awx.conf.license import get_license -from awx.main.utils import get_awx_version, get_ansible_version +from awx.main.utils import get_awx_version from awx.main.analytics.collectors import ( counts, instance_info, @@ -127,7 +127,6 @@ def metrics(): 'insights_analytics': str(settings.INSIGHTS_TRACKING_STATE), 'tower_url_base': settings.TOWER_URL_BASE, 'tower_version': get_awx_version(), - 'ansible_version': get_ansible_version(), 'license_type': license_info.get('license_type', 'UNLICENSED'), 'license_expiry': str(license_info.get('time_remaining', 0)), 'pendo_tracking': settings.PENDO_TRACKING_STATE, diff --git a/awx/main/utils/common.py b/awx/main/utils/common.py index b40f7fb270..8ad4a9f485 100644 --- a/awx/main/utils/common.py +++ b/awx/main/utils/common.py @@ -44,7 +44,6 @@ __all__ = [ 'underscore_to_camelcase', 'memoize', 'memoize_delete', - 'get_ansible_version', 'get_licenser', 'get_awx_http_client_headers', 'get_awx_version', @@ -192,20 +191,6 @@ def memoize_delete(function_name): return cache.delete(function_name) -@memoize() -def get_ansible_version(): - """ - Return Ansible version installed. - Ansible path needs to be provided to account for custom virtual environments - """ - try: - proc = subprocess.Popen(['ansible', '--version'], stdout=subprocess.PIPE) - result = smart_str(proc.communicate()[0]) - return result.split('\n')[0].replace('ansible', '').strip() - except Exception: - return 'unknown' - - def get_awx_version(): """ Return AWX version as reported by setuptools. diff --git a/awx/ui_next/src/components/About/About.jsx b/awx/ui_next/src/components/About/About.jsx index f68f75b613..db6fbd782d 100644 --- a/awx/ui_next/src/components/About/About.jsx +++ b/awx/ui_next/src/components/About/About.jsx @@ -2,17 +2,12 @@ import React from 'react'; import PropTypes from 'prop-types'; import { withI18n } from '@lingui/react'; import { t } from '@lingui/macro'; -import { - AboutModal, - TextContent, - TextList, - TextListItem, -} from '@patternfly/react-core'; +import { AboutModal } from '@patternfly/react-core'; import { BrandName } from '../../variables'; import brandLogoImg from './brand-logo.svg'; -function About({ ansible_version, version, isOpen, onClose, i18n }) { +function About({ version, isOpen, onClose, i18n }) { const createSpeechBubble = () => { let text = `${BrandName} ${version}`; let top = ''; @@ -52,27 +47,17 @@ function About({ ansible_version, version, isOpen, onClose, i18n }) { || || `} - - - - {i18n._(t`Ansible Version`)} - - {ansible_version} - - ); } About.propTypes = { - ansible_version: PropTypes.string, isOpen: PropTypes.bool, onClose: PropTypes.func.isRequired, version: PropTypes.string, }; About.defaultProps = { - ansible_version: null, isOpen: false, version: null, }; diff --git a/awx/ui_next/src/components/AppContainer/AppContainer.jsx b/awx/ui_next/src/components/AppContainer/AppContainer.jsx index 102d61ac26..6c4016ac9b 100644 --- a/awx/ui_next/src/components/AppContainer/AppContainer.jsx +++ b/awx/ui_next/src/components/AppContainer/AppContainer.jsx @@ -204,7 +204,6 @@ function AppContainer({ i18n, navRouteConfig = [], children }) { {isReady && {children}} ', () => { - const ansible_version = '111'; const version = '222'; beforeEach(() => { ConfigAPI.read.mockResolvedValue({ data: { - ansible_version, version, }, }); @@ -93,7 +91,6 @@ describe('', () => { // check about modal content const content = await waitForElement(wrapper, aboutModalContent); - expect(content.find('dd').text()).toContain(ansible_version); expect(content.find('pre').text()).toContain(`< AWX ${version} >`); // close about modal From b110a4a94e06fd4a9175c0cd1fc47302d35bd5fe Mon Sep 17 00:00:00 2001 From: nixocio Date: Thu, 18 Mar 2021 16:54:56 -0400 Subject: [PATCH 14/51] Add EE to the settings page Allow a system admin to set the global default execution environment. See: https://github.com/ansible/awx/issues/9088 This PR is also addressing the issue: https://github.com/ansible/awx/issues/9669 --- awx/main/conf.py | 2 +- .../Lookup/ExecutionEnvironmentLookup.jsx | 24 ++++-- .../MiscSystemDetail/MiscSystemDetail.jsx | 17 +++-- .../MiscSystemDetail.test.jsx | 42 ++++++++++- .../MiscSystemEdit/MiscSystemEdit.jsx | 75 +++++++++++++++++-- .../MiscSystemEdit/MiscSystemEdit.test.jsx | 75 ++++++++++++++++++- .../screens/Setting/shared/SettingDetail.jsx | 14 +--- .../shared/data.allSettingOptions.json | 19 ++++- .../Setting/shared/data.allSettings.json | 3 +- 9 files changed, 233 insertions(+), 38 deletions(-) diff --git a/awx/main/conf.py b/awx/main/conf.py index dbeaec9040..5cfd2977f7 100644 --- a/awx/main/conf.py +++ b/awx/main/conf.py @@ -186,7 +186,7 @@ register( default=None, queryset=ExecutionEnvironment.objects.all(), label=_('Global default execution environment'), - help_text=_('.'), + help_text=_('The Execution Environment to be used when one has not been configured for a job template.'), category=_('System'), category_slug='system', ) diff --git a/awx/ui_next/src/components/Lookup/ExecutionEnvironmentLookup.jsx b/awx/ui_next/src/components/Lookup/ExecutionEnvironmentLookup.jsx index 4647d5809e..b3134cb527 100644 --- a/awx/ui_next/src/components/Lookup/ExecutionEnvironmentLookup.jsx +++ b/awx/ui_next/src/components/Lookup/ExecutionEnvironmentLookup.jsx @@ -25,6 +25,7 @@ function ExecutionEnvironmentLookup({ globallyAvailable, i18n, isDefaultEnvironment, + isGlobalDefaultEnvironment, isDisabled, onBlur, onChange, @@ -154,17 +155,26 @@ function ExecutionEnvironmentLookup({ ); + const renderLabel = ( + globalDefaultEnvironment, + defaultExecutionEnvironment + ) => { + if (globalDefaultEnvironment) { + return i18n._(t`Global Default Execution Environment`); + } + if (defaultExecutionEnvironment) { + return i18n._(t`Default Execution Environment`); + } + return i18n._(t`Execution Environment`); + }; + return ( } > - {isDisabled ? ( + {tooltip ? ( {renderLookup()} ) : ( renderLookup() @@ -180,6 +190,7 @@ ExecutionEnvironmentLookup.propTypes = { popoverContent: string, onChange: func.isRequired, isDefaultEnvironment: bool, + isGlobalDefaultEnvironment: bool, projectId: oneOfType([number, string]), organizationId: oneOfType([number, string]), }; @@ -187,6 +198,7 @@ ExecutionEnvironmentLookup.propTypes = { ExecutionEnvironmentLookup.defaultProps = { popoverContent: '', isDefaultEnvironment: false, + isGlobalDefaultEnvironment: false, value: null, projectId: null, organizationId: null, diff --git a/awx/ui_next/src/screens/Setting/MiscSystem/MiscSystemDetail/MiscSystemDetail.jsx b/awx/ui_next/src/screens/Setting/MiscSystem/MiscSystemDetail/MiscSystemDetail.jsx index 02f37b2d96..54eac90e9f 100644 --- a/awx/ui_next/src/screens/Setting/MiscSystem/MiscSystemDetail/MiscSystemDetail.jsx +++ b/awx/ui_next/src/screens/Setting/MiscSystem/MiscSystemDetail/MiscSystemDetail.jsx @@ -9,7 +9,7 @@ import ContentError from '../../../../components/ContentError'; import ContentLoading from '../../../../components/ContentLoading'; import { DetailList } from '../../../../components/DetailList'; import RoutedTabs from '../../../../components/RoutedTabs'; -import { SettingsAPI } from '../../../../api'; +import { SettingsAPI, ExecutionEnvironmentsAPI } from '../../../../api'; import useRequest from '../../../../util/useRequest'; import { useConfig } from '../../../../contexts/Config'; import { useSettings } from '../../../../contexts/Settings'; @@ -23,7 +23,15 @@ function MiscSystemDetail({ i18n }) { const { isLoading, error, request, result: system } = useRequest( useCallback(async () => { const { data } = await SettingsAPI.readCategory('all'); - + let DEFAULT_EXECUTION_ENVIRONMENT = ''; + if (data.DEFAULT_EXECUTION_ENVIRONMENT) { + const { + data: { name }, + } = await ExecutionEnvironmentsAPI.readDetail( + data.DEFAULT_EXECUTION_ENVIRONMENT + ); + DEFAULT_EXECUTION_ENVIRONMENT = name; + } const { OAUTH2_PROVIDER: { ACCESS_TOKEN_EXPIRE_SECONDS, @@ -49,19 +57,17 @@ function MiscSystemDetail({ i18n }) { 'SESSION_COOKIE_AGE', 'TOWER_URL_BASE' ); - const systemData = { ...pluckedSystemData, ACCESS_TOKEN_EXPIRE_SECONDS, REFRESH_TOKEN_EXPIRE_SECONDS, AUTHORIZATION_CODE_EXPIRE_SECONDS, + DEFAULT_EXECUTION_ENVIRONMENT, }; - const { OAUTH2_PROVIDER: OAUTH2_PROVIDER_OPTIONS, ...options } = allOptions; - const systemOptions = { ...options, ACCESS_TOKEN_EXPIRE_SECONDS: { @@ -80,7 +86,6 @@ function MiscSystemDetail({ i18n }) { label: i18n._(t`Authorization Code Expiration`), }, }; - const mergedData = {}; Object.keys(systemData).forEach(key => { mergedData[key] = systemOptions[key]; diff --git a/awx/ui_next/src/screens/Setting/MiscSystem/MiscSystemDetail/MiscSystemDetail.test.jsx b/awx/ui_next/src/screens/Setting/MiscSystem/MiscSystemDetail/MiscSystemDetail.test.jsx index aa8b2e334d..6fbebb6ab8 100644 --- a/awx/ui_next/src/screens/Setting/MiscSystem/MiscSystemDetail/MiscSystemDetail.test.jsx +++ b/awx/ui_next/src/screens/Setting/MiscSystem/MiscSystemDetail/MiscSystemDetail.test.jsx @@ -5,7 +5,7 @@ import { waitForElement, } from '../../../../../testUtils/enzymeHelpers'; import { SettingsProvider } from '../../../../contexts/Settings'; -import { SettingsAPI } from '../../../../api'; +import { SettingsAPI, ExecutionEnvironmentsAPI } from '../../../../api'; import { assertDetail, assertVariableDetail, @@ -14,13 +14,14 @@ import mockAllOptions from '../../shared/data.allSettingOptions.json'; import MiscSystemDetail from './MiscSystemDetail'; jest.mock('../../../../api/models/Settings'); +jest.mock('../../../../api/models/ExecutionEnvironments'); + SettingsAPI.readCategory.mockResolvedValue({ data: { ALLOW_OAUTH2_FOR_EXTERNAL_USERS: false, AUTH_BASIC_ENABLED: true, AUTOMATION_ANALYTICS_GATHER_INTERVAL: 14400, AUTOMATION_ANALYTICS_URL: 'https://example.com', - CUSTOM_VENV_PATHS: [], INSIGHTS_TRACKING_STATE: false, LOGIN_REDIRECT_OVERRIDE: 'https://redirect.com', MANAGE_ORGANIZATION_AUTH: true, @@ -36,6 +37,16 @@ SettingsAPI.readCategory.mockResolvedValue({ SESSIONS_PER_USER: -1, SESSION_COOKIE_AGE: 30000000000, TOWER_URL_BASE: 'https://towerhost', + DEFAULT_EXECUTION_ENVIRONMENT: 1, + }, +}); + +ExecutionEnvironmentsAPI.readDetail.mockResolvedValue({ + data: { + id: 1, + name: 'Foo', + image: 'quay.io/ansible/awx-ee', + pull: 'missing', }, }); @@ -110,6 +121,33 @@ describe('', () => { assertDetail(wrapper, 'Red Hat customer username', 'mock name'); assertDetail(wrapper, 'Refresh Token Expiration', '3 seconds'); assertVariableDetail(wrapper, 'Remote Host Headers', '[]'); + assertDetail(wrapper, 'Global default execution environment', 'Foo'); + }); + + test('should render execution environment as not configured', async () => { + ExecutionEnvironmentsAPI.readDetail.mockResolvedValue({ + data: {}, + }); + let newWrapper; + await act(async () => { + newWrapper = mountWithContexts( + + + + ); + }); + await waitForElement(newWrapper, 'ContentLoading', el => el.length === 0); + + assertDetail( + newWrapper, + 'Global default execution environment', + 'Not configured' + ); }); test('should hide edit button from non-superusers', async () => { diff --git a/awx/ui_next/src/screens/Setting/MiscSystem/MiscSystemEdit/MiscSystemEdit.jsx b/awx/ui_next/src/screens/Setting/MiscSystem/MiscSystemEdit/MiscSystemEdit.jsx index bb19b52f21..5411326eb0 100644 --- a/awx/ui_next/src/screens/Setting/MiscSystem/MiscSystemEdit/MiscSystemEdit.jsx +++ b/awx/ui_next/src/screens/Setting/MiscSystem/MiscSystemEdit/MiscSystemEdit.jsx @@ -9,6 +9,7 @@ import ContentError from '../../../../components/ContentError'; import ContentLoading from '../../../../components/ContentLoading'; import { FormSubmitError } from '../../../../components/FormField'; import { FormColumnLayout } from '../../../../components/FormLayout'; +import { ExecutionEnvironmentLookup } from '../../../../components/Lookup'; import { useSettings } from '../../../../contexts/Settings'; import { BooleanField, @@ -20,7 +21,7 @@ import { } from '../../shared'; import useModal from '../../../../util/useModal'; import useRequest from '../../../../util/useRequest'; -import { SettingsAPI } from '../../../../api'; +import { SettingsAPI, ExecutionEnvironmentsAPI } from '../../../../api'; import { pluck, formatJson } from '../../shared/settingUtils'; function MiscSystemEdit({ i18n }) { @@ -44,7 +45,6 @@ function MiscSystemEdit({ i18n }) { 'AUTH_BASIC_ENABLED', 'AUTOMATION_ANALYTICS_GATHER_INTERVAL', 'AUTOMATION_ANALYTICS_URL', - 'CUSTOM_VENV_PATHS', 'INSIGHTS_TRACKING_STATE', 'LOGIN_REDIRECT_OVERRIDE', 'MANAGE_ORGANIZATION_AUTH', @@ -55,7 +55,8 @@ function MiscSystemEdit({ i18n }) { 'REMOTE_HOST_HEADERS', 'SESSIONS_PER_USER', 'SESSION_COOKIE_AGE', - 'TOWER_URL_BASE' + 'TOWER_URL_BASE', + 'DEFAULT_EXECUTION_ENVIRONMENT' ); const systemData = { @@ -128,6 +129,7 @@ function MiscSystemEdit({ i18n }) { AUTHORIZATION_CODE_EXPIRE_SECONDS, ...formData } = form; + await submitForm({ ...formData, REMOTE_HOST_HEADERS: formatJson(formData.REMOTE_HOST_HEADERS), @@ -136,6 +138,8 @@ function MiscSystemEdit({ i18n }) { REFRESH_TOKEN_EXPIRE_SECONDS, AUTHORIZATION_CODE_EXPIRE_SECONDS, }, + DEFAULT_EXECUTION_ENVIRONMENT: + formData.DEFAULT_EXECUTION_ENVIRONMENT?.id || null, }); }; @@ -178,16 +182,73 @@ function MiscSystemEdit({ i18n }) { return acc; }, {}); + const executionEnvironmentId = + system?.DEFAULT_EXECUTION_ENVIRONMENT?.value || null; + + const { + isLoading: isLoadingExecutionEnvironment, + error: errorExecutionEnvironment, + request: fetchExecutionEnvironment, + result: executionEnvironment, + } = useRequest( + useCallback(async () => { + if (!executionEnvironmentId) { + return ''; + } + const { data } = await ExecutionEnvironmentsAPI.readDetail( + executionEnvironmentId + ); + return data; + }, [executionEnvironmentId]) + ); + + useEffect(() => { + fetchExecutionEnvironment(); + }, [fetchExecutionEnvironment]); + return ( - {isLoading && } - {!isLoading && error && } - {!isLoading && system && ( - + {(isLoading || isLoadingExecutionEnvironment) && } + {!(isLoading || isLoadingExecutionEnvironment) && error && ( + + )} + {!(isLoading || isLoadingExecutionEnvironment) && system && ( + {formik => { return (
+ + formik.setFieldTouched('DEFAULT_EXECUTION_ENVIRONMENT') + } + value={formik.values.DEFAULT_EXECUTION_ENVIRONMENT} + onChange={value => + formik.setFieldValue( + 'DEFAULT_EXECUTION_ENVIRONMENT', + value + ) + } + popoverContent={i18n._( + t`The Execution Environment to be used when one has not been configured for a job template.` + )} + isGlobalDefaultEnvironment + /> ', () => { let wrapper; let history; @@ -42,10 +83,40 @@ describe('', () => { await waitForElement(wrapper, 'ContentLoading', el => el.length === 0); }); - test('initially renders without crashing', () => { + test('initially renders without crashing', async () => { expect(wrapper.find('MiscSystemEdit').length).toBe(1); }); + test('save button should call updateAll', async () => { + expect(wrapper.find('MiscSystemEdit').length).toBe(1); + + wrapper.find('ExecutionEnvironmentLookup').invoke('onChange')({ + id: 1, + name: 'Foo', + }); + wrapper.update(); + await act(async () => { + wrapper.find('button[aria-label="Save"]').simulate('click'); + }); + wrapper.update(); + expect(SettingsAPI.updateAll).toHaveBeenCalledWith(systemData); + }); + + test('should remove execution environment', async () => { + expect(wrapper.find('MiscSystemEdit').length).toBe(1); + + wrapper.find('ExecutionEnvironmentLookup').invoke('onChange')(null); + wrapper.update(); + await act(async () => { + wrapper.find('button[aria-label="Save"]').simulate('click'); + }); + + expect(SettingsAPI.updateAll).toHaveBeenCalledWith({ + ...systemData, + DEFAULT_EXECUTION_ENVIRONMENT: null, + }); + }); + test('should successfully send default values to api on form revert all', async () => { expect(SettingsAPI.updateAll).toHaveBeenCalledTimes(0); expect(wrapper.find('RevertAllAlert')).toHaveLength(0); diff --git a/awx/ui_next/src/screens/Setting/shared/SettingDetail.jsx b/awx/ui_next/src/screens/Setting/shared/SettingDetail.jsx index d58c89e721..fc6498b67e 100644 --- a/awx/ui_next/src/screens/Setting/shared/SettingDetail.jsx +++ b/awx/ui_next/src/screens/Setting/shared/SettingDetail.jsx @@ -88,6 +88,8 @@ export default withI18n()( ); break; case 'choice': + case 'field': + case 'string': detail = ( ); break; - case 'string': - detail = ( - - ); - break; default: detail = null; } diff --git a/awx/ui_next/src/screens/Setting/shared/data.allSettingOptions.json b/awx/ui_next/src/screens/Setting/shared/data.allSettingOptions.json index 43713bf1fa..ac885ced92 100644 --- a/awx/ui_next/src/screens/Setting/shared/data.allSettingOptions.json +++ b/awx/ui_next/src/screens/Setting/shared/data.allSettingOptions.json @@ -2944,7 +2944,15 @@ "child": { "type": "field" } - } + }, + "DEFAULT_EXECUTION_ENVIRONMENT": { + "type": "field", + "label": "Global default execution environment", + "help_text": "The Execution Environment to be used when one has not been configured for a job template.", + "category": "System", + "category_slug": "system", + "defined_in_file": false + } }, "PUT": { "ACTIVITY_STREAM_ENABLED": { @@ -7049,6 +7057,15 @@ "read_only": false } }, + "DEFAULT_EXECUTION_ENVIRONMENT": { + "type": "field", + "required": false, + "label": "Global default execution environment", + "help_text": "The Execution Environment to be used when one has not been configured for a job template.", + "category": "System", + "category_slug": "system", + "default": null + }, "SOCIAL_AUTH_SAML_TEAM_ATTR": { "type": "nested object", "required": false, diff --git a/awx/ui_next/src/screens/Setting/shared/data.allSettings.json b/awx/ui_next/src/screens/Setting/shared/data.allSettings.json index 81d32ea5ac..57b810615c 100644 --- a/awx/ui_next/src/screens/Setting/shared/data.allSettings.json +++ b/awx/ui_next/src/screens/Setting/shared/data.allSettings.json @@ -303,5 +303,6 @@ "applications":{"fields":["name"],"adj_list":[["organization","organizations"]]}, "users":{"fields":["username"],"adj_list":[]}, "instances":{"fields":["hostname"],"adj_list":[]} - } + }, + "DEFAULT_EXECUTION_ENVIRONMENT": 1 } From 81266cf7a7c5e8abfa3b91997a0d7570e0a8067f Mon Sep 17 00:00:00 2001 From: Ryan Petrello Date: Thu, 25 Mar 2021 13:25:25 -0400 Subject: [PATCH 15/51] only run black on files added or modified in the commit --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 3813380cf8..6e5e2ee34c 100644 --- a/Makefile +++ b/Makefile @@ -276,7 +276,7 @@ black: reports (set -o pipefail && $@ $(BLACK_ARGS) awx awxkit awx_collection | tee reports/$@.report) .git/hooks/pre-commit: - echo "[ -z \$$AWX_IGNORE_BLACK ] && (black --check \`git diff --cached --name-only | grep -E '\.py$\'\` || (echo 'To fix this, run \`make black\` to auto-format your code prior to commit, or set AWX_IGNORE_BLACK=1' && exit 1))" > .git/hooks/pre-commit + echo "[ -z \$$AWX_IGNORE_BLACK ] && (black --check \`git diff --cached --name-only --diff-filter=AM | grep -E '\.py$\'\` || (echo 'To fix this, run \`make black\` to auto-format your code prior to commit, or set AWX_IGNORE_BLACK=1' && exit 1))" > .git/hooks/pre-commit chmod +x .git/hooks/pre-commit genschema: reports From 8bb90dde3312b58bc08821987ddc8bacd20ba386 Mon Sep 17 00:00:00 2001 From: Shane McDonald Date: Thu, 25 Mar 2021 13:27:19 -0400 Subject: [PATCH 16/51] Devel images are now on Quay --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 3813380cf8..8afaebf041 100644 --- a/Makefile +++ b/Makefile @@ -23,7 +23,7 @@ VENV_BASE ?= /var/lib/awx/venv/ SCL_PREFIX ?= CELERY_SCHEDULE_FILE ?= /var/lib/awx/beat.db -DEV_DOCKER_TAG_BASE ?= gcr.io/ansible-tower-engineering +DEV_DOCKER_TAG_BASE ?= quay.io/awx DEVEL_IMAGE_NAME ?= $(DEV_DOCKER_TAG_BASE)/awx_devel:$(COMPOSE_TAG) # Python packages to install only from source (not from binary wheels) From 3f342feadd552129a2de7e9013a3376fcd96ce6d Mon Sep 17 00:00:00 2001 From: Seth Foster Date: Tue, 23 Mar 2021 17:38:48 -0400 Subject: [PATCH 17/51] Fix api/v2/metrics data displaying incorrect value - Use a locally defined prometheus registry instead of global registry --- awx/main/analytics/metrics.py | 228 ++++++++++++++++++---------------- 1 file changed, 118 insertions(+), 110 deletions(-) diff --git a/awx/main/analytics/metrics.py b/awx/main/analytics/metrics.py index e889719ded..3399dcadee 100644 --- a/awx/main/analytics/metrics.py +++ b/awx/main/analytics/metrics.py @@ -1,5 +1,5 @@ from django.conf import settings -from prometheus_client import REGISTRY, PROCESS_COLLECTOR, PLATFORM_COLLECTOR, GC_COLLECTOR, Gauge, Info, generate_latest +from prometheus_client import PROCESS_COLLECTOR, PLATFORM_COLLECTOR, GC_COLLECTOR, CollectorRegistry, Gauge, Info, generate_latest from awx.conf.license import get_license from awx.main.utils import get_awx_version, get_ansible_version @@ -11,115 +11,123 @@ from awx.main.analytics.collectors import ( ) -REGISTRY.unregister(PROCESS_COLLECTOR) -REGISTRY.unregister(PLATFORM_COLLECTOR) -REGISTRY.unregister(GC_COLLECTOR) - -SYSTEM_INFO = Info('awx_system', 'AWX System Information') -ORG_COUNT = Gauge('awx_organizations_total', 'Number of organizations') -USER_COUNT = Gauge('awx_users_total', 'Number of users') -TEAM_COUNT = Gauge('awx_teams_total', 'Number of teams') -INV_COUNT = Gauge('awx_inventories_total', 'Number of inventories') -PROJ_COUNT = Gauge('awx_projects_total', 'Number of projects') -JT_COUNT = Gauge('awx_job_templates_total', 'Number of job templates') -WFJT_COUNT = Gauge('awx_workflow_job_templates_total', 'Number of workflow job templates') -HOST_COUNT = Gauge( - 'awx_hosts_total', - 'Number of hosts', - [ - 'type', - ], -) -SCHEDULE_COUNT = Gauge('awx_schedules_total', 'Number of schedules') -INV_SCRIPT_COUNT = Gauge('awx_inventory_scripts_total', 'Number of invetory scripts') -USER_SESSIONS = Gauge( - 'awx_sessions_total', - 'Number of sessions', - [ - 'type', - ], -) -CUSTOM_VENVS = Gauge('awx_custom_virtualenvs_total', 'Number of virtualenvs') -RUNNING_JOBS = Gauge('awx_running_jobs_total', 'Number of running jobs on the Tower system') -PENDING_JOBS = Gauge('awx_pending_jobs_total', 'Number of pending jobs on the Tower system') -STATUS = Gauge( - 'awx_status_total', - 'Status of Job launched', - [ - 'status', - ], -) - -INSTANCE_CAPACITY = Gauge( - 'awx_instance_capacity', - 'Capacity of each node in a Tower system', - [ - 'hostname', - 'instance_uuid', - ], -) -INSTANCE_CPU = Gauge( - 'awx_instance_cpu', - 'CPU cores on each node in a Tower system', - [ - 'hostname', - 'instance_uuid', - ], -) -INSTANCE_MEMORY = Gauge( - 'awx_instance_memory', - 'RAM (Kb) on each node in a Tower system', - [ - 'hostname', - 'instance_uuid', - ], -) -INSTANCE_INFO = Info( - 'awx_instance', - 'Info about each node in a Tower system', - [ - 'hostname', - 'instance_uuid', - ], -) -INSTANCE_LAUNCH_TYPE = Gauge( - 'awx_instance_launch_type_total', - 'Type of Job launched', - [ - 'node', - 'launch_type', - ], -) -INSTANCE_STATUS = Gauge( - 'awx_instance_status_total', - 'Status of Job launched', - [ - 'node', - 'status', - ], -) -INSTANCE_CONSUMED_CAPACITY = Gauge( - 'awx_instance_consumed_capacity', - 'Consumed capacity of each node in a Tower system', - [ - 'hostname', - 'instance_uuid', - ], -) -INSTANCE_REMAINING_CAPACITY = Gauge( - 'awx_instance_remaining_capacity', - 'Remaining capacity of each node in a Tower system', - [ - 'hostname', - 'instance_uuid', - ], -) - -LICENSE_INSTANCE_TOTAL = Gauge('awx_license_instance_total', 'Total number of managed hosts provided by your license') -LICENSE_INSTANCE_FREE = Gauge('awx_license_instance_free', 'Number of remaining managed hosts provided by your license') - - def metrics(): + REGISTRY = CollectorRegistry() + + SYSTEM_INFO = Info('awx_system', 'AWX System Information', registry=REGISTRY) + ORG_COUNT = Gauge('awx_organizations_total', 'Number of organizations', registry=REGISTRY) + USER_COUNT = Gauge('awx_users_total', 'Number of users', registry=REGISTRY) + TEAM_COUNT = Gauge('awx_teams_total', 'Number of teams', registry=REGISTRY) + INV_COUNT = Gauge('awx_inventories_total', 'Number of inventories', registry=REGISTRY) + PROJ_COUNT = Gauge('awx_projects_total', 'Number of projects', registry=REGISTRY) + JT_COUNT = Gauge('awx_job_templates_total', 'Number of job templates', registry=REGISTRY) + WFJT_COUNT = Gauge('awx_workflow_job_templates_total', 'Number of workflow job templates', registry=REGISTRY) + HOST_COUNT = Gauge( + 'awx_hosts_total', + 'Number of hosts', + [ + 'type', + ], + registry=REGISTRY, + ) + SCHEDULE_COUNT = Gauge('awx_schedules_total', 'Number of schedules', registry=REGISTRY) + INV_SCRIPT_COUNT = Gauge('awx_inventory_scripts_total', 'Number of invetory scripts', registry=REGISTRY) + USER_SESSIONS = Gauge( + 'awx_sessions_total', + 'Number of sessions', + [ + 'type', + ], + registry=REGISTRY, + ) + CUSTOM_VENVS = Gauge('awx_custom_virtualenvs_total', 'Number of virtualenvs', registry=REGISTRY) + RUNNING_JOBS = Gauge('awx_running_jobs_total', 'Number of running jobs on the Tower system', registry=REGISTRY) + PENDING_JOBS = Gauge('awx_pending_jobs_total', 'Number of pending jobs on the Tower system', registry=REGISTRY) + STATUS = Gauge( + 'awx_status_total', + 'Status of Job launched', + [ + 'status', + ], + registry=REGISTRY, + ) + + INSTANCE_CAPACITY = Gauge( + 'awx_instance_capacity', + 'Capacity of each node in a Tower system', + [ + 'hostname', + 'instance_uuid', + ], + registry=REGISTRY, + ) + INSTANCE_CPU = Gauge( + 'awx_instance_cpu', + 'CPU cores on each node in a Tower system', + [ + 'hostname', + 'instance_uuid', + ], + registry=REGISTRY, + ) + INSTANCE_MEMORY = Gauge( + 'awx_instance_memory', + 'RAM (Kb) on each node in a Tower system', + [ + 'hostname', + 'instance_uuid', + ], + registry=REGISTRY, + ) + INSTANCE_INFO = Info( + 'awx_instance', + 'Info about each node in a Tower system', + [ + 'hostname', + 'instance_uuid', + ], + registry=REGISTRY, + ) + INSTANCE_LAUNCH_TYPE = Gauge( + 'awx_instance_launch_type_total', + 'Type of Job launched', + [ + 'node', + 'launch_type', + ], + registry=REGISTRY, + ) + INSTANCE_STATUS = Gauge( + 'awx_instance_status_total', + 'Status of Job launched', + [ + 'node', + 'status', + ], + registry=REGISTRY, + ) + INSTANCE_CONSUMED_CAPACITY = Gauge( + 'awx_instance_consumed_capacity', + 'Consumed capacity of each node in a Tower system', + [ + 'hostname', + 'instance_uuid', + ], + registry=REGISTRY, + ) + INSTANCE_REMAINING_CAPACITY = Gauge( + 'awx_instance_remaining_capacity', + 'Remaining capacity of each node in a Tower system', + [ + 'hostname', + 'instance_uuid', + ], + registry=REGISTRY, + ) + + LICENSE_INSTANCE_TOTAL = Gauge('awx_license_instance_total', 'Total number of managed hosts provided by your license', registry=REGISTRY) + LICENSE_INSTANCE_FREE = Gauge('awx_license_instance_free', 'Number of remaining managed hosts provided by your license', registry=REGISTRY) + license_info = get_license() SYSTEM_INFO.info( { @@ -197,7 +205,7 @@ def metrics(): for status, value in statuses.items(): INSTANCE_STATUS.labels(node=node, status=status).set(value) - return generate_latest() + return generate_latest(registry=REGISTRY) __all__ = ['metrics'] From f8a698d12727692eb96b8fc59d4dfa3f13fc86f5 Mon Sep 17 00:00:00 2001 From: Shane McDonald Date: Thu, 25 Mar 2021 15:21:01 -0400 Subject: [PATCH 18/51] Update minikube.md --- docs/development/minikube.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/development/minikube.md b/docs/development/minikube.md index 93ad0d20a1..73bb85e2ae 100644 --- a/docs/development/minikube.md +++ b/docs/development/minikube.md @@ -66,7 +66,7 @@ In the root of awx-operator: ``` $ ansible-playbook ansible/instantiate-awx-deployment.yml \ -e development_mode=yes \ - -e tower_image=gcr.io/ansible-tower-engineering/awx_kube_devel:devel \ + -e tower_image=quay.io/awx/awx_kube_devel:devel \ -e tower_image_pull_policy=Always \ -e tower_ingress_type=ingress ``` @@ -81,7 +81,7 @@ In the root of the AWX repo: ``` $ make awx-kube-dev-build -$ docker push gcr.io/ansible-tower-engineering/awx_kube_devel:${COMPOSE_TAG} +$ docker push quay.io/awx/awx_kube_devel:${COMPOSE_TAG} ``` In the root of awx-operator: @@ -89,7 +89,7 @@ In the root of awx-operator: ``` $ ansible-playbook ansible/instantiate-awx-deployment.yml \ -e development_mode=yes \ - -e tower_image=gcr.io/ansible-tower-engineering/awx_kube_devel:${COMPOSE_TAG} \ + -e tower_image=quay.io/awx/awx_kube_devel:${COMPOSE_TAG} \ -e tower_image_pull_policy=Always \ -e tower_ingress_type=ingress ``` From 0c569c67fd81c5593eeba4a5e1c6acf5b65106d6 Mon Sep 17 00:00:00 2001 From: Seth Foster Date: Tue, 23 Mar 2021 16:05:10 -0400 Subject: [PATCH 19/51] Add subsystem metrics - Adds a Metrics() class that can track data such as number of events the callback receiver inserted into database - Exposes this metric data at the api/v2/metrics/ endpoint. This data is prometheus-friendly - Metric data is stored in memory, then periodically saved to Redis. - Metric data is periodically broadcast to other nodes in the cluster, so that each node has a copy of the most recent metric data collected. --- awx/api/renderers.py | 14 +- awx/api/templates/api/metrics_view.md | 1 + awx/api/views/metrics.py | 8 +- awx/main/analytics/analytics_tasks.py | 14 + awx/main/analytics/subsystem_metrics.py | 304 ++++++++++++++++++ awx/main/consumers.py | 1 - awx/main/dispatch/worker/callback.py | 39 ++- awx/main/queue.py | 3 +- awx/main/tasks.py | 2 + .../functional/analytics/test_metrics.py | 26 +- awx/main/wsbroadcast.py | 8 +- awx/settings/defaults.py | 10 + 12 files changed, 410 insertions(+), 20 deletions(-) create mode 100644 awx/api/templates/api/metrics_view.md create mode 100644 awx/main/analytics/analytics_tasks.py create mode 100644 awx/main/analytics/subsystem_metrics.py diff --git a/awx/api/renderers.py b/awx/api/renderers.py index 48cba6cf5c..d19d6ee318 100644 --- a/awx/api/renderers.py +++ b/awx/api/renderers.py @@ -129,6 +129,18 @@ class PrometheusJSONRenderer(renderers.JSONRenderer): parsed_metrics = text_string_to_metric_families(data) data = {} for family in parsed_metrics: + data[family.name] = {} + data[family.name]['help_text'] = family.documentation + data[family.name]['type'] = family.type + data[family.name]['samples'] = [] for sample in family.samples: - data[sample[0]] = {"labels": sample[1], "value": sample[2]} + sample_dict = {"labels": sample[1], "value": sample[2]} + if family.type == 'histogram': + if sample[0].endswith("_sum"): + sample_dict['sample_type'] = "sum" + elif sample[0].endswith("_count"): + sample_dict['sample_type'] = "count" + elif sample[0].endswith("_bucket"): + sample_dict['sample_type'] = "bucket" + data[family.name]['samples'].append(sample_dict) return super(PrometheusJSONRenderer, self).render(data, accepted_media_type, renderer_context) diff --git a/awx/api/templates/api/metrics_view.md b/awx/api/templates/api/metrics_view.md new file mode 100644 index 0000000000..dbc4d2e043 --- /dev/null +++ b/awx/api/templates/api/metrics_view.md @@ -0,0 +1 @@ +query params to filter response, e.g., ?subsystemonly=1&metric=callback_receiver_events_insert_db&node=awx-1 diff --git a/awx/api/views/metrics.py b/awx/api/views/metrics.py index dd40f11900..212acf3890 100644 --- a/awx/api/views/metrics.py +++ b/awx/api/views/metrics.py @@ -14,6 +14,7 @@ from rest_framework.exceptions import PermissionDenied # AWX # from awx.main.analytics import collectors +import awx.main.analytics.subsystem_metrics as s_metrics from awx.main.analytics.metrics import metrics from awx.api import renderers @@ -33,5 +34,10 @@ class MetricsView(APIView): def get(self, request): ''' Show Metrics Details ''' if request.user.is_superuser or request.user.is_system_auditor: - return Response(metrics().decode('UTF-8')) + metrics_to_show = '' + if not request.query_params.get('subsystemonly', "0") == "1": + metrics_to_show += metrics().decode('UTF-8') + if not request.query_params.get('dbonly', "0") == "1": + metrics_to_show += s_metrics.metrics(request) + return Response(metrics_to_show) raise PermissionDenied() diff --git a/awx/main/analytics/analytics_tasks.py b/awx/main/analytics/analytics_tasks.py new file mode 100644 index 0000000000..990cacfafb --- /dev/null +++ b/awx/main/analytics/analytics_tasks.py @@ -0,0 +1,14 @@ +# Python +import logging + +# AWX +from awx.main.analytics.subsystem_metrics import Metrics +from awx.main.dispatch.publish import task +from awx.main.dispatch import get_local_queuename + +logger = logging.getLogger('awx.main.scheduler') + + +@task(queue=get_local_queuename) +def send_subsystem_metrics(): + Metrics().send_metrics() diff --git a/awx/main/analytics/subsystem_metrics.py b/awx/main/analytics/subsystem_metrics.py new file mode 100644 index 0000000000..b5ecf39e90 --- /dev/null +++ b/awx/main/analytics/subsystem_metrics.py @@ -0,0 +1,304 @@ +import redis +import json +import time +import logging + +from django.conf import settings +from django.apps import apps +from awx.main.consumers import emit_channel_notification + +root_key = 'awx_metrics' +logger = logging.getLogger('awx.main.wsbroadcast') + + +class BaseM: + def __init__(self, field, help_text): + self.field = field + self.help_text = help_text + self.current_value = 0 + + def clear_value(self, conn): + conn.hset(root_key, self.field, 0) + self.current_value = 0 + + def inc(self, value): + self.current_value += value + + def set(self, value): + self.current_value = value + + def decode(self, conn): + value = conn.hget(root_key, self.field) + return self.decode_value(value) + + def to_prometheus(self, instance_data): + output_text = f"# HELP {self.field} {self.help_text}\n# TYPE {self.field} gauge\n" + for instance in instance_data: + output_text += f'{self.field}{{node="{instance}"}} {instance_data[instance][self.field]}\n' + return output_text + + +class FloatM(BaseM): + def decode_value(self, value): + if value is not None: + return float(value) + else: + return 0.0 + + def store_value(self, conn): + conn.hincrbyfloat(root_key, self.field, self.current_value) + self.current_value = 0 + + +class IntM(BaseM): + def decode_value(self, value): + if value is not None: + return int(value) + else: + return 0 + + def store_value(self, conn): + conn.hincrby(root_key, self.field, self.current_value) + self.current_value = 0 + + +class SetIntM(BaseM): + def decode_value(self, value): + if value is not None: + return int(value) + else: + return 0 + + def store_value(self, conn): + # do not set value if it has not changed since last time this was called + if self.current_value is not None: + conn.hset(root_key, self.field, self.current_value) + self.current_value = None + + +class SetFloatM(SetIntM): + def decode_value(self, value): + if value is not None: + return float(value) + else: + return 0 + + +class HistogramM(BaseM): + def __init__(self, field, help_text, buckets): + self.buckets = buckets + self.buckets_to_keys = {} + for b in buckets: + self.buckets_to_keys[b] = IntM(field + '_' + str(b), '') + self.inf = IntM(field + '_inf', '') + self.sum = IntM(field + '_sum', '') + super(HistogramM, self).__init__(field, help_text) + + def clear_value(self, conn): + conn.hset(root_key, self.field, 0) + self.inf.clear_value(conn) + self.sum.clear_value(conn) + for b in self.buckets_to_keys.values(): + b.clear_value(conn) + super(HistogramM, self).clear_value(conn) + + def observe(self, value): + for b in self.buckets: + if value <= b: + self.buckets_to_keys[b].inc(1) + break + self.sum.inc(value) + self.inf.inc(1) + + def decode(self, conn): + values = {'counts': []} + for b in self.buckets_to_keys: + values['counts'].append(self.buckets_to_keys[b].decode(conn)) + values['sum'] = self.sum.decode(conn) + values['inf'] = self.inf.decode(conn) + return values + + def store_value(self, conn): + for b in self.buckets: + self.buckets_to_keys[b].store_value(conn) + self.sum.store_value(conn) + self.inf.store_value(conn) + + def to_prometheus(self, instance_data): + output_text = f"# HELP {self.field} {self.help_text}\n# TYPE {self.field} histogram\n" + for instance in instance_data: + for i, b in enumerate(self.buckets): + output_text += f'{self.field}_bucket{{le="{b}",node="{instance}"}} {sum(instance_data[instance][self.field]["counts"][0:i+1])}\n' + output_text += f'{self.field}_bucket{{le="+Inf",node="{instance}"}} {instance_data[instance][self.field]["inf"]}\n' + output_text += f'{self.field}_count{{node="{instance}"}} {instance_data[instance][self.field]["inf"]}\n' + output_text += f'{self.field}_sum{{node="{instance}"}} {instance_data[instance][self.field]["sum"]}\n' + return output_text + + +class Metrics: + def __init__(self, auto_pipe_execute=True): + self.pipe = redis.Redis.from_url(settings.BROKER_URL).pipeline() + self.conn = redis.Redis.from_url(settings.BROKER_URL) + self.last_pipe_execute = time.time() + # track if metrics have been modified since last saved to redis + # start with True so that we get an initial save to redis + self.metrics_have_changed = True + self.pipe_execute_interval = settings.SUBSYSTEM_METRICS_INTERVAL_SAVE_TO_REDIS + self.send_metrics_interval = settings.SUBSYSTEM_METRICS_INTERVAL_SEND_METRICS + # auto pipe execute will commit transaction of metric data to redis + # at a regular interval (pipe_execute_interval). If set to False, + # the calling function should call .pipe_execute() explicitly + self.auto_pipe_execute = auto_pipe_execute + Instance = apps.get_model('main', 'Instance') + self.instance_name = Instance.objects.me().hostname + + # metric name, help_text + METRICSLIST = [ + SetIntM('callback_receiver_events_queue_size_redis', 'Current number of events in redis queue'), + IntM('callback_receiver_events_popped_redis', 'Number of events popped from redis'), + IntM('callback_receiver_events_in_memory', 'Current number of events in memory (in transfer from redis to db)'), + IntM('callback_receiver_batch_events_errors', 'Number of times batch insertion failed'), + FloatM('callback_receiver_events_insert_db_seconds', 'Time spent saving events to database'), + IntM('callback_receiver_events_insert_db', 'Number of events batch inserted into database'), + HistogramM( + 'callback_receiver_batch_events_insert_db', 'Number of events batch inserted into database', settings.SUBSYSTEM_METRICS_BATCH_INSERT_BUCKETS + ), + FloatM('subsystem_metrics_pipe_execute_seconds', 'Time spent saving metrics to redis'), + IntM('subsystem_metrics_pipe_execute_calls', 'Number of calls to pipe_execute'), + FloatM('subsystem_metrics_send_metrics_seconds', 'Time spent sending metrics to other nodes'), + ] + # turn metric list into dictionary with the metric name as a key + self.METRICS = {} + for m in METRICSLIST: + self.METRICS[m.field] = m + + # track last time metrics were sent to other nodes + self.previous_send_metrics = SetFloatM('send_metrics_time', 'Timestamp of previous send_metrics call') + + def clear_values(self): + for m in self.METRICS.values(): + m.clear_value(self.conn) + self.metrics_have_changed = True + self.conn.delete(root_key + "_lock") + + def inc(self, field, value): + if value != 0: + self.METRICS[field].inc(value) + self.metrics_have_changed = True + if self.auto_pipe_execute is True and self.should_pipe_execute() is True: + self.pipe_execute() + + def set(self, field, value): + self.METRICS[field].set(value) + self.metrics_have_changed = True + if self.auto_pipe_execute is True and self.should_pipe_execute() is True: + self.pipe_execute() + + def observe(self, field, value): + self.METRICS[field].observe(value) + self.metrics_have_changed = True + if self.auto_pipe_execute is True and self.should_pipe_execute() is True: + self.pipe_execute() + + def serialize_local_metrics(self): + data = self.load_local_metrics() + return json.dumps(data) + + def load_local_metrics(self): + # generate python dictionary of key values from metrics stored in redis + data = {} + for field in self.METRICS: + data[field] = self.METRICS[field].decode(self.conn) + return data + + def store_metrics(self, data_json): + # called when receiving metrics from other instances + data = json.loads(data_json) + if self.instance_name != data['instance']: + logger.debug(f"{self.instance_name} received subsystem metrics from {data['instance']}") + self.conn.set(root_key + "_instance_" + data['instance'], data['metrics']) + + def should_pipe_execute(self): + if self.metrics_have_changed is False: + return False + if time.time() - self.last_pipe_execute > self.pipe_execute_interval: + return True + else: + return False + + def pipe_execute(self): + if self.metrics_have_changed is True: + duration_to_save = time.perf_counter() + for m in self.METRICS: + self.METRICS[m].store_value(self.pipe) + self.pipe.execute() + self.last_pipe_execute = time.time() + self.metrics_have_changed = False + duration_to_save = time.perf_counter() - duration_to_save + self.METRICS['subsystem_metrics_pipe_execute_seconds'].inc(duration_to_save) + self.METRICS['subsystem_metrics_pipe_execute_calls'].inc(1) + + duration_to_save = time.perf_counter() + self.send_metrics() + duration_to_save = time.perf_counter() - duration_to_save + self.METRICS['subsystem_metrics_send_metrics_seconds'].inc(duration_to_save) + + def send_metrics(self): + # more than one thread could be calling this at the same time, so should + # get acquire redis lock before sending metrics + lock = self.conn.lock(root_key + '_lock', thread_local=False) + if not lock.acquire(blocking=False): + return + try: + current_time = time.time() + if current_time - self.previous_send_metrics.decode(self.conn) > self.send_metrics_interval: + payload = { + 'instance': self.instance_name, + 'metrics': self.serialize_local_metrics(), + } + # store a local copy as well + self.store_metrics(json.dumps(payload)) + emit_channel_notification("metrics", payload) + self.previous_send_metrics.set(current_time) + self.previous_send_metrics.store_value(self.conn) + finally: + lock.release() + + def load_other_metrics(self, request): + # data received from other nodes are stored in their own keys + # e.g., awx_metrics_instance_awx-1, awx_metrics_instance_awx-2 + # this method looks for keys with "_instance_" in the name and loads the data + # also filters data based on request query params + # if additional filtering is added, update metrics_view.md + instances_filter = request.query_params.getlist("node") + # get a sorted list of instance names + instance_names = [self.instance_name] + for m in self.conn.scan_iter(root_key + '_instance_*'): + instance_names.append(m.decode('UTF-8').split('_instance_')[1]) + instance_names.sort() + # load data, including data from the this local instance + instance_data = {} + for instance in instance_names: + if len(instances_filter) == 0 or instance in instances_filter: + instance_data_from_redis = self.conn.get(root_key + '_instance_' + instance) + # data from other instances may not be available. That is OK. + if instance_data_from_redis: + instance_data[instance] = json.loads(instance_data_from_redis.decode('UTF-8')) + return instance_data + + def generate_metrics(self, request): + # takes the api request, filters, and generates prometheus data + # if additional filtering is added, update metrics_view.md + instance_data = self.load_other_metrics(request) + metrics_filter = request.query_params.getlist("metric") + output_text = '' + if instance_data: + for field in self.METRICS: + if len(metrics_filter) == 0 or field in metrics_filter: + output_text += self.METRICS[field].to_prometheus(instance_data) + return output_text + + +def metrics(request): + m = Metrics() + return m.generate_metrics(request) diff --git a/awx/main/consumers.py b/awx/main/consumers.py index a2425ec337..21ebe9d771 100644 --- a/awx/main/consumers.py +++ b/awx/main/consumers.py @@ -13,7 +13,6 @@ from channels.generic.websocket import AsyncJsonWebsocketConsumer from channels.layers import get_channel_layer from channels.db import database_sync_to_async - logger = logging.getLogger('awx.main.consumers') XRF_KEY = '_auth_user_xrf' diff --git a/awx/main/dispatch/worker/callback.py b/awx/main/dispatch/worker/callback.py index 68b8d5fd4f..acfb0bce02 100644 --- a/awx/main/dispatch/worker/callback.py +++ b/awx/main/dispatch/worker/callback.py @@ -20,7 +20,7 @@ from awx.main.models import JobEvent, AdHocCommandEvent, ProjectUpdateEvent, Inv from awx.main.tasks import handle_success_and_failure_notifications from awx.main.models.events import emit_event_detail from awx.main.utils.profiling import AWXProfiler - +import awx.main.analytics.subsystem_metrics as s_metrics from .base import BaseWorker logger = logging.getLogger('awx.main.commands.run_callback_receiver') @@ -46,16 +46,22 @@ class CallbackBrokerWorker(BaseWorker): self.buff = {} self.pid = os.getpid() self.redis = redis.Redis.from_url(settings.BROKER_URL) + self.subsystem_metrics = s_metrics.Metrics(auto_pipe_execute=False) + self.queue_pop = 0 + self.queue_name = settings.CALLBACK_QUEUE self.prof = AWXProfiler("CallbackBrokerWorker") for key in self.redis.keys('awx_callback_receiver_statistics_*'): self.redis.delete(key) def read(self, queue): try: - res = self.redis.blpop(settings.CALLBACK_QUEUE, timeout=1) + res = self.redis.blpop(self.queue_name, timeout=1) if res is None: return {'event': 'FLUSH'} self.total += 1 + self.queue_pop += 1 + self.subsystem_metrics.inc('callback_receiver_events_popped_redis', 1) + self.subsystem_metrics.inc('callback_receiver_events_in_memory', 1) return json.loads(res[1]) except redis.exceptions.RedisError: logger.exception("encountered an error communicating with redis") @@ -64,8 +70,19 @@ class CallbackBrokerWorker(BaseWorker): logger.exception("failed to decode JSON message from redis") finally: self.record_statistics() + self.record_read_metrics() + return {'event': 'FLUSH'} + def record_read_metrics(self): + if self.queue_pop == 0: + return + if self.subsystem_metrics.should_pipe_execute() is True: + queue_size = self.redis.llen(self.queue_name) + self.subsystem_metrics.set('callback_receiver_events_queue_size_redis', queue_size) + self.subsystem_metrics.pipe_execute() + self.queue_pop = 0 + def record_statistics(self): # buffer stat recording to once per (by default) 5s if time.time() - self.last_stats > settings.JOB_EVENT_STATISTICS_INTERVAL: @@ -99,27 +116,44 @@ class CallbackBrokerWorker(BaseWorker): def flush(self, force=False): now = tz_now() if force or (time.time() - self.last_flush) > settings.JOB_EVENT_BUFFER_SECONDS or any([len(events) >= 1000 for events in self.buff.values()]): + bulk_events_saved = 0 + singular_events_saved = 0 + metrics_events_batch_save_errors = 0 for cls, events in self.buff.items(): logger.debug(f'{cls.__name__}.objects.bulk_create({len(events)})') for e in events: if not e.created: e.created = now e.modified = now + duration_to_save = time.perf_counter() try: cls.objects.bulk_create(events) + bulk_events_saved += len(events) except Exception: # if an exception occurs, we should re-attempt to save the # events one-by-one, because something in the list is # broken/stale + metrics_events_batch_save_errors += 1 for e in events: try: e.save() + singular_events_saved += 1 except Exception: logger.exception('Database Error Saving Job Event') + duration_to_save = time.perf_counter() - duration_to_save for e in events: emit_event_detail(e) self.buff = {} self.last_flush = time.time() + # only update metrics if we saved events + if (bulk_events_saved + singular_events_saved) > 0: + self.subsystem_metrics.inc('callback_receiver_batch_events_errors', metrics_events_batch_save_errors) + self.subsystem_metrics.inc('callback_receiver_events_insert_db_seconds', duration_to_save) + self.subsystem_metrics.inc('callback_receiver_events_insert_db', bulk_events_saved + singular_events_saved) + self.subsystem_metrics.observe('callback_receiver_batch_events_insert_db', bulk_events_saved) + self.subsystem_metrics.inc('callback_receiver_events_in_memory', -(bulk_events_saved + singular_events_saved)) + if self.subsystem_metrics.should_pipe_execute() is True: + self.subsystem_metrics.pipe_execute() def perform_work(self, body): try: @@ -169,6 +203,7 @@ class CallbackBrokerWorker(BaseWorker): except Exception: logger.exception('Worker failed to emit notifications: Job {}'.format(job_identifier)) finally: + self.subsystem_metrics.inc('callback_receiver_events_in_memory', -1) GuidMiddleware.set_guid('') return diff --git a/awx/main/queue.py b/awx/main/queue.py index 88fc2c8288..ebac0622e4 100644 --- a/awx/main/queue.py +++ b/awx/main/queue.py @@ -8,7 +8,7 @@ import redis # Django from django.conf import settings - +import awx.main.analytics.subsystem_metrics as s_metrics __all__ = ['CallbackQueueDispatcher'] @@ -28,6 +28,7 @@ class CallbackQueueDispatcher(object): self.queue = getattr(settings, 'CALLBACK_QUEUE', '') self.logger = logging.getLogger('awx.main.queue.CallbackQueueDispatcher') self.connection = redis.Redis.from_url(settings.BROKER_URL) + self.subsystem_metrics = s_metrics.Metrics() def dispatch(self, obj): self.connection.rpush(self.queue, json.dumps(obj, cls=AnsibleJSONEncoder)) diff --git a/awx/main/tasks.py b/awx/main/tasks.py index de1b15377b..7945b8e275 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -107,6 +107,7 @@ from awx.main.consumers import emit_channel_notification from awx.main import analytics from awx.conf import settings_registry from awx.conf.license import get_license +from awx.main.analytics.subsystem_metrics import Metrics from rest_framework.exceptions import PermissionDenied @@ -170,6 +171,7 @@ def dispatch_startup(): cluster_node_heartbeat() if Instance.objects.me().is_controller(): awx_isolated_heartbeat() + Metrics().clear_values() # Update Tower's rsyslog.conf file based on loggins settings in the db reconfigure_rsyslog() diff --git a/awx/main/tests/functional/analytics/test_metrics.py b/awx/main/tests/functional/analytics/test_metrics.py index 94076d1362..442c83699c 100644 --- a/awx/main/tests/functional/analytics/test_metrics.py +++ b/awx/main/tests/functional/analytics/test_metrics.py @@ -56,24 +56,28 @@ def test_metrics_counts(organization_factory, job_template_factory, workflow_job assert EXPECTED_VALUES[name] == value +def get_metrics_view_db_only(): + return reverse('api:metrics_view') + '?dbonly=1' + + @pytest.mark.django_db def test_metrics_permissions(get, admin, org_admin, alice, bob, organization): - assert get(reverse('api:metrics_view'), user=admin).status_code == 200 - assert get(reverse('api:metrics_view'), user=org_admin).status_code == 403 - assert get(reverse('api:metrics_view'), user=alice).status_code == 403 - assert get(reverse('api:metrics_view'), user=bob).status_code == 403 + assert get(get_metrics_view_db_only(), user=admin).status_code == 200 + assert get(get_metrics_view_db_only(), user=org_admin).status_code == 403 + assert get(get_metrics_view_db_only(), user=alice).status_code == 403 + assert get(get_metrics_view_db_only(), user=bob).status_code == 403 organization.auditor_role.members.add(bob) - assert get(reverse('api:metrics_view'), user=bob).status_code == 403 + assert get(get_metrics_view_db_only(), user=bob).status_code == 403 Role.singleton('system_auditor').members.add(bob) bob.is_system_auditor = True - assert get(reverse('api:metrics_view'), user=bob).status_code == 200 + assert get(get_metrics_view_db_only(), user=bob).status_code == 200 @pytest.mark.django_db def test_metrics_http_methods(get, post, patch, put, options, admin): - assert get(reverse('api:metrics_view'), user=admin).status_code == 200 - assert put(reverse('api:metrics_view'), user=admin).status_code == 405 - assert patch(reverse('api:metrics_view'), user=admin).status_code == 405 - assert post(reverse('api:metrics_view'), user=admin).status_code == 405 - assert options(reverse('api:metrics_view'), user=admin).status_code == 200 + assert get(get_metrics_view_db_only(), user=admin).status_code == 200 + assert put(get_metrics_view_db_only(), user=admin).status_code == 405 + assert patch(get_metrics_view_db_only(), user=admin).status_code == 405 + assert post(get_metrics_view_db_only(), user=admin).status_code == 405 + assert options(get_metrics_view_db_only(), user=admin).status_code == 200 diff --git a/awx/main/wsbroadcast.py b/awx/main/wsbroadcast.py index e2ee9fc431..184ae06122 100644 --- a/awx/main/wsbroadcast.py +++ b/awx/main/wsbroadcast.py @@ -15,7 +15,7 @@ from awx.main.analytics.broadcast_websocket import ( BroadcastWebsocketStats, BroadcastWebsocketStatsManager, ) - +import awx.main.analytics.subsystem_metrics as s_metrics logger = logging.getLogger('awx.main.wsbroadcast') @@ -68,6 +68,7 @@ class WebsocketTask: self.protocol = protocol self.verify_ssl = verify_ssl self.channel_layer = None + self.subsystem_metrics = s_metrics.Metrics() async def run_loop(self, websocket: aiohttp.ClientWebSocketResponse): raise RuntimeError("Implement me") @@ -144,9 +145,10 @@ class BroadcastWebsocketTask(WebsocketTask): logmsg = "{} {}".format(logmsg, payload) logger.warn(logmsg) continue - (group, message) = unwrap_broadcast_msg(payload) - + if group == "metrics": + self.subsystem_metrics.store_metrics(message) + continue await self.channel_layer.group_send(group, {"type": "internal.message", "text": message}) diff --git a/awx/settings/defaults.py b/awx/settings/defaults.py index e51f66007d..2daa33d4b3 100644 --- a/awx/settings/defaults.py +++ b/awx/settings/defaults.py @@ -224,6 +224,15 @@ JOB_EVENT_MAX_QUEUE_SIZE = 10000 # The number of job events to migrate per-transaction when moving from int -> bigint JOB_EVENT_MIGRATION_CHUNK_SIZE = 1000000 +# Histogram buckets for the callback_receiver_batch_events_insert_db metric +SUBSYSTEM_METRICS_BATCH_INSERT_BUCKETS = [10, 50, 150, 350, 650, 2000] + +# Interval in seconds for sending local metrics to other nodes +SUBSYSTEM_METRICS_INTERVAL_SEND_METRICS = 3 + +# Interval in seconds for saving local metrics to redis +SUBSYSTEM_METRICS_INTERVAL_SAVE_TO_REDIS = 2 + # The maximum allowed jobs to start on a given task manager cycle START_TASK_LIMIT = 100 @@ -427,6 +436,7 @@ CELERYBEAT_SCHEDULE = { 'gather_analytics': {'task': 'awx.main.tasks.gather_analytics', 'schedule': timedelta(minutes=5)}, 'task_manager': {'task': 'awx.main.scheduler.tasks.run_task_manager', 'schedule': timedelta(seconds=20), 'options': {'expires': 20}}, 'k8s_reaper': {'task': 'awx.main.tasks.awx_k8s_reaper', 'schedule': timedelta(seconds=60), 'options': {'expires': 50}}, + 'send_subsystem_metrics': {'task': 'awx.main.analytics.analytics_tasks.send_subsystem_metrics', 'schedule': timedelta(seconds=20)}, # 'isolated_heartbeat': set up at the end of production.py and development.py } From d3eb66b6fe52d7da1855c6a4d885ac344f9cf580 Mon Sep 17 00:00:00 2001 From: nixocio Date: Wed, 24 Mar 2021 12:45:16 -0400 Subject: [PATCH 20/51] Update RBAC for EE Update RBAC for EE details page. See: https://github.com/ansible/awx/issues/9416 --- .../ExecutionEnvironmentDetails.jsx | 46 ++++++----- .../ExecutionEnvironmentDetails.test.jsx | 78 ++++++++++++++++++- 2 files changed, 102 insertions(+), 22 deletions(-) diff --git a/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentDetails/ExecutionEnvironmentDetails.jsx b/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentDetails/ExecutionEnvironmentDetails.jsx index a5c89f8d89..35e4209fc8 100644 --- a/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentDetails/ExecutionEnvironmentDetails.jsx +++ b/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentDetails/ExecutionEnvironmentDetails.jsx @@ -110,27 +110,31 @@ function ExecutionEnvironmentDetails({ executionEnvironment, i18n }) { {!managedByTower && ( - - - {i18n._(t`Delete`)} - + {summary_fields.user_capabilities?.edit && ( + + )} + {summary_fields.user_capabilities?.delete && ( + + {i18n._(t`Delete`)} + + )} )} diff --git a/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentDetails/ExecutionEnvironmentDetails.test.jsx b/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentDetails/ExecutionEnvironmentDetails.test.jsx index ce0bf830ed..19dd4e1e16 100644 --- a/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentDetails/ExecutionEnvironmentDetails.test.jsx +++ b/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentDetails/ExecutionEnvironmentDetails.test.jsx @@ -2,7 +2,10 @@ import React from 'react'; import { act } from 'react-dom/test-utils'; import { createMemoryHistory } from 'history'; -import { mountWithContexts } from '../../../../testUtils/enzymeHelpers'; +import { + mountWithContexts, + waitForElement, +} from '../../../../testUtils/enzymeHelpers'; import { ExecutionEnvironmentsAPI } from '../../../api'; import ExecutionEnvironmentDetails from './ExecutionEnvironmentDetails'; @@ -22,6 +25,11 @@ const executionEnvironment = { credential: '/api/v2/credentials/4/', }, summary_fields: { + user_capabilities: { + edit: true, + delete: true, + copy: true, + }, credential: { id: 4, name: 'Container Registry', @@ -175,6 +183,7 @@ describe('', () => { expect(wrapper.find('Button[aria-label="Delete"]')).toHaveLength(0); }); + test('should have proper number of delete detail requests', async () => { const history = createMemoryHistory({ initialEntries: ['/execution_environments/42/details'], @@ -193,4 +202,71 @@ describe('', () => { wrapper.find('DeleteButton').prop('deleteDetailsRequests') ).toHaveLength(4); }); + + test('should show edit button for users with edit permission', async () => { + await act(async () => { + wrapper = mountWithContexts( + + ); + }); + const editButton = await waitForElement( + wrapper, + 'ExecutionEnvironmentDetails Button[aria-label="edit"]' + ); + expect(editButton.text()).toEqual('Edit'); + expect(editButton.prop('to')).toBe('/execution_environments/17/edit'); + }); + + test('should hide edit button for users without edit permission', async () => { + await act(async () => { + wrapper = mountWithContexts( + + ); + }); + await waitForElement(wrapper, 'ExecutionEnvironmentDetails'); + expect( + wrapper.find('ExecutionEnvironmentDetails Button[aria-label="edit"]') + .length + ).toBe(0); + }); + + test('should show delete button for users with delete permission', async () => { + await act(async () => { + wrapper = mountWithContexts( + + ); + }); + const deleteButton = await waitForElement( + wrapper, + 'ExecutionEnvironmentDetails Button[aria-label="Delete"]' + ); + expect(deleteButton.text()).toEqual('Delete'); + }); + + test('should hide delete button for users without delete permission', async () => { + await act(async () => { + wrapper = mountWithContexts( + + ); + }); + await waitForElement(wrapper, 'ExecutionEnvironmentDetails'); + expect( + wrapper.find('ExecutionEnvironmentDetails Button[aria-label="Delete"]') + .length + ).toBe(0); + }); }); From a1aec29e487e864cfb5b5e6fc036dd9f8e39ddd1 Mon Sep 17 00:00:00 2001 From: Jim Ladd Date: Thu, 25 Mar 2021 13:17:21 -0700 Subject: [PATCH 21/51] let jupyter install ipython --- requirements/requirements_dev.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements/requirements_dev.txt b/requirements/requirements_dev.txt index df8cc1cb13..2c16fad0c5 100644 --- a/requirements/requirements_dev.txt +++ b/requirements/requirements_dev.txt @@ -1,7 +1,6 @@ django-debug-toolbar==1.11 django-rest-swagger pprofile -ipython==5.2.1 unittest2 black pytest From a86196cfa36a6f5a4572a80ca432df3c4f845b44 Mon Sep 17 00:00:00 2001 From: Jake McDermott Date: Thu, 25 Mar 2021 17:40:56 -0400 Subject: [PATCH 22/51] Don't append a slash to file paths --- .../shared/InventorySourceSubForms/SCMSubForm.jsx | 7 ------- .../shared/InventorySourceSubForms/SCMSubForm.test.jsx | 4 ++-- 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/awx/ui_next/src/screens/Inventory/shared/InventorySourceSubForms/SCMSubForm.jsx b/awx/ui_next/src/screens/Inventory/shared/InventorySourceSubForms/SCMSubForm.jsx index a16949776a..f00d77e5c4 100644 --- a/awx/ui_next/src/screens/Inventory/shared/InventorySourceSubForms/SCMSubForm.jsx +++ b/awx/ui_next/src/screens/Inventory/shared/InventorySourceSubForms/SCMSubForm.jsx @@ -122,9 +122,6 @@ const SCMSubForm = ({ autoPopulateProject, i18n }) => { onSelect={(event, value) => { setIsOpen(false); value = value.trim(); - if (!value.endsWith('/')) { - value += '/'; - } sourcePathHelpers.setValue(value); }} aria-label={i18n._(t`Select source path`)} @@ -132,10 +129,6 @@ const SCMSubForm = ({ autoPopulateProject, i18n }) => { isCreatable onCreateOption={value => { value.trim(); - - if (!value.endsWith('/')) { - value += '/'; - } setSourcePath([...sourcePath, value]); }} > diff --git a/awx/ui_next/src/screens/Inventory/shared/InventorySourceSubForms/SCMSubForm.test.jsx b/awx/ui_next/src/screens/Inventory/shared/InventorySourceSubForms/SCMSubForm.test.jsx index 0d3c47f451..715d0df7cd 100644 --- a/awx/ui_next/src/screens/Inventory/shared/InventorySourceSubForms/SCMSubForm.test.jsx +++ b/awx/ui_next/src/screens/Inventory/shared/InventorySourceSubForms/SCMSubForm.test.jsx @@ -98,7 +98,7 @@ describe('', () => { }); wrapper.update(); expect(wrapper.find('Select#source_path').prop('selections')).toEqual( - 'bar/' + 'bar' ); await act(async () => { @@ -138,7 +138,7 @@ describe('', () => { customWrapper.find('Select').invoke('onSelect')({}, 'newPath'); }); customWrapper.update(); - expect(customWrapper.find('Select').prop('selections')).toBe('newPath/'); + expect(customWrapper.find('Select').prop('selections')).toBe('newPath'); }); test('Update on project update should be disabled', async () => { const customInitialValues = { From 93d1df4e4bd36ae5208f03692adb501a1b4135e2 Mon Sep 17 00:00:00 2001 From: Jake McDermott Date: Thu, 25 Mar 2021 18:31:45 -0400 Subject: [PATCH 23/51] Use custom text for setting source path --- .../Inventory/shared/InventorySourceSubForms/SCMSubForm.jsx | 1 + 1 file changed, 1 insertion(+) diff --git a/awx/ui_next/src/screens/Inventory/shared/InventorySourceSubForms/SCMSubForm.jsx b/awx/ui_next/src/screens/Inventory/shared/InventorySourceSubForms/SCMSubForm.jsx index f00d77e5c4..e874157d99 100644 --- a/awx/ui_next/src/screens/Inventory/shared/InventorySourceSubForms/SCMSubForm.jsx +++ b/awx/ui_next/src/screens/Inventory/shared/InventorySourceSubForms/SCMSubForm.jsx @@ -126,6 +126,7 @@ const SCMSubForm = ({ autoPopulateProject, i18n }) => { }} aria-label={i18n._(t`Select source path`)} placeholder={i18n._(t`Select source path`)} + createText={i18n._(t`Set source path to`)} isCreatable onCreateOption={value => { value.trim(); From 81024f8dfea5abf9a2bfaee2d9a59c5d0e64dce8 Mon Sep 17 00:00:00 2001 From: beeankha Date: Fri, 26 Mar 2021 11:37:35 -0400 Subject: [PATCH 24/51] Remove/modify usage of tower-cli in Collections tests --- awx/main/tasks.py | 2 +- .../tasks/create_project_dir.yml | 18 +++--------------- 2 files changed, 4 insertions(+), 16 deletions(-) diff --git a/awx/main/tasks.py b/awx/main/tasks.py index 7945b8e275..b5cb7ec1e8 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -2106,7 +2106,7 @@ class RunProjectUpdate(BaseTask): d = super(RunProjectUpdate, self).get_password_prompts(passwords) d[r'Username for.*:\s*?$'] = 'scm_username' d[r'Password for.*:\s*?$'] = 'scm_password' - d['Password:\s*?$'] = 'scm_password' # noqa + d[r'Password:\s*?$'] = 'scm_password' d[r'\S+?@\S+?\'s\s+?password:\s*?$'] = 'scm_password' d[r'Enter passphrase for .*:\s*?$'] = 'scm_key_unlock' d[r'Bad passphrase, try again for .*:\s*?$'] = '' diff --git a/awx_collection/tests/integration/targets/tower_project_manual/tasks/create_project_dir.yml b/awx_collection/tests/integration/targets/tower_project_manual/tasks/create_project_dir.yml index 5388a6d3fd..d92dee8221 100644 --- a/awx_collection/tests/integration/targets/tower_project_manual/tasks/create_project_dir.yml +++ b/awx_collection/tests/integration/targets/tower_project_manual/tasks/create_project_dir.yml @@ -1,16 +1,4 @@ --- -- name: get tower host variable - shell: tower-cli config host | cut -d ' ' -f2 - register: host - -- name: get tower username variable - shell: tower-cli config username | cut -d ' ' -f2 - register: username - -- name: get tower password variable - shell: tower-cli config password | cut -d ' ' -f2 - register: password - - name: Fetch project_base_dir uri: url: "{{ host.stdout }}/api/v2/config/" @@ -44,15 +32,15 @@ organization: Default - name: Disable process isolation - command: tower-cli setting modify AWX_PROOT_ENABLED false + command: awx-cli setting modify AWX_PROOT_ENABLED false - block: - name: Create a directory for manual project vars: project_base_dir: "{{ awx_config.json.project_base_dir }}" - command: tower-cli ad_hoc launch --wait --inventory localhost + command: awx-cli ad_hoc launch --wait --inventory localhost --credential dummy --module-name command --module-args "mkdir -p {{ project_base_dir }}/{{ project_dir_name }}" always: - name: enable process isolation - command: tower-cli setting modify AWX_PROOT_ENABLED true + command: awx-cli setting modify AWX_PROOT_ENABLED true From f38c9e74787678c8868b530bf16b50cc3edd17e6 Mon Sep 17 00:00:00 2001 From: beeankha Date: Fri, 26 Mar 2021 15:35:52 -0400 Subject: [PATCH 25/51] Update manual project Collection integration test to be compatible with EEs --- .../plugins/module_utils/tower_api.py | 2 +- .../tasks/create_project_dir.yml | 47 +++++++++++-------- .../template_galaxy/templates/README.md.j2 | 2 +- 3 files changed, 30 insertions(+), 21 deletions(-) diff --git a/awx_collection/plugins/module_utils/tower_api.py b/awx_collection/plugins/module_utils/tower_api.py index 2edba2b502..da56ad1d26 100644 --- a/awx_collection/plugins/module_utils/tower_api.py +++ b/awx_collection/plugins/module_utils/tower_api.py @@ -272,7 +272,7 @@ class TowerAPIModule(TowerModule): if self._COLLECTION_TYPE not in self.collection_to_version or self.collection_to_version[self._COLLECTION_TYPE] != tower_type: self.warn("You are using the {0} version of this collection but connecting to {1}".format(self._COLLECTION_TYPE, tower_type)) elif collection_compare_ver != tower_compare_ver: - self.warn("You are running collection version {0} but connecting to tower version {1}".format(self._COLLECTION_VERSION, tower_version)) + self.warn("You are running collection version {0} but connecting to {2} version {1}".format(self._COLLECTION_VERSION, tower_version, tower_type)) self.version_checked = True diff --git a/awx_collection/tests/integration/targets/tower_project_manual/tasks/create_project_dir.yml b/awx_collection/tests/integration/targets/tower_project_manual/tasks/create_project_dir.yml index d92dee8221..ef15df8c50 100644 --- a/awx_collection/tests/integration/targets/tower_project_manual/tasks/create_project_dir.yml +++ b/awx_collection/tests/integration/targets/tower_project_manual/tasks/create_project_dir.yml @@ -1,13 +1,9 @@ --- -- name: Fetch project_base_dir - uri: - url: "{{ host.stdout }}/api/v2/config/" - user: "{{ username.stdout }}" - password: "{{ password.stdout }}" - validate_certs: false - return_content: true - force_basic_auth: true - register: awx_config +- name: Load the UI settings + set_fact: + project_base_dir: "{{ tower_settings.project_base_dir }}" + vars: + tower_settings: "{{ lookup('awx.awx.tower_api', 'config/') }}" - tower_inventory: name: localhost @@ -31,16 +27,29 @@ -----END EC PRIVATE KEY----- organization: Default -- name: Disable process isolation - command: awx-cli setting modify AWX_PROOT_ENABLED false - - block: + - name: Add a path to a setting + tower_settings: + name: AWX_ISOLATION_SHOW_PATHS + value: "[{{ project_base_dir }}]" + - name: Create a directory for manual project - vars: - project_base_dir: "{{ awx_config.json.project_base_dir }}" - command: awx-cli ad_hoc launch --wait --inventory localhost - --credential dummy --module-name command - --module-args "mkdir -p {{ project_base_dir }}/{{ project_dir_name }}" + tower_ad_hoc_command: + credential: dummy + inventory: localhost + job_type: run + module_args: "mkdir -p {{ project_base_dir }}/{{ project_dir_name }}" + module_name: command + wait: True + always: - - name: enable process isolation - command: awx-cli setting modify AWX_PROOT_ENABLED true + - name: Delete path from setting + tower_settings: + name: AWX_ISOLATION_SHOW_PATHS + value: [] + + - name: Delete dummy credential + tower_credential: + name: dummy + kind: ssh + state: absent diff --git a/awx_collection/tools/roles/template_galaxy/templates/README.md.j2 b/awx_collection/tools/roles/template_galaxy/templates/README.md.j2 index ed02006c3d..274df392b5 100644 --- a/awx_collection/tools/roles/template_galaxy/templates/README.md.j2 +++ b/awx_collection/tools/roles/template_galaxy/templates/README.md.j2 @@ -127,7 +127,7 @@ py.test awx_collection/test/awx/ ## Running Integration Tests -The integration tests require a virtualenv with `ansible` >= 2.9 and `tower_cli`. +The integration tests require a virtualenv with `ansible` >= 2.9 and `awxkit`. The collection must first be installed, which can be done using `make install_collection`. You also need a configuration file, as described in the running section. From b681d1078f412217b8f8e83e52c8986c151d796d Mon Sep 17 00:00:00 2001 From: beeankha Date: Fri, 26 Mar 2021 16:00:53 -0400 Subject: [PATCH 26/51] Update unit test to pull in product names that are no longer hardcoded --- awx_collection/plugins/module_utils/tower_api.py | 4 +++- awx_collection/test/awx/test_module_utils.py | 4 ++-- .../targets/tower_project_manual/tasks/create_project_dir.yml | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/awx_collection/plugins/module_utils/tower_api.py b/awx_collection/plugins/module_utils/tower_api.py index da56ad1d26..f6c63b08de 100644 --- a/awx_collection/plugins/module_utils/tower_api.py +++ b/awx_collection/plugins/module_utils/tower_api.py @@ -272,7 +272,9 @@ class TowerAPIModule(TowerModule): if self._COLLECTION_TYPE not in self.collection_to_version or self.collection_to_version[self._COLLECTION_TYPE] != tower_type: self.warn("You are using the {0} version of this collection but connecting to {1}".format(self._COLLECTION_TYPE, tower_type)) elif collection_compare_ver != tower_compare_ver: - self.warn("You are running collection version {0} but connecting to {2} version {1}".format(self._COLLECTION_VERSION, tower_version, tower_type)) + self.warn( + "You are running collection version {0} but connecting to {2} version {1}".format(self._COLLECTION_VERSION, tower_version, tower_type) + ) self.version_checked = True diff --git a/awx_collection/test/awx/test_module_utils.py b/awx_collection/test/awx/test_module_utils.py index 8f2a8e52d5..89bd44154e 100644 --- a/awx_collection/test/awx/test_module_utils.py +++ b/awx_collection/test/awx/test_module_utils.py @@ -59,7 +59,7 @@ def test_version_warning(collection_import, silence_warning): my_module._COLLECTION_TYPE = "awx" my_module.get_endpoint('ping') silence_warning.assert_called_once_with( - 'You are running collection version {} but connecting to tower version {}'.format(my_module._COLLECTION_VERSION, ping_version) + 'You are running collection version {} but connecting to {} version {}'.format(my_module._COLLECTION_VERSION, awx_name, ping_version) ) @@ -107,7 +107,7 @@ def test_version_warning_strictness_tower(collection_import, silence_warning): my_module._COLLECTION_TYPE = "tower" my_module.get_endpoint('ping') silence_warning.assert_called_once_with( - 'You are running collection version {} but connecting to tower version {}'.format(my_module._COLLECTION_VERSION, ping_version) + 'You are running collection version {} but connecting to {} version {}'.format(my_module._COLLECTION_VERSION, tower_name, ping_version) ) diff --git a/awx_collection/tests/integration/targets/tower_project_manual/tasks/create_project_dir.yml b/awx_collection/tests/integration/targets/tower_project_manual/tasks/create_project_dir.yml index ef15df8c50..807c604dd9 100644 --- a/awx_collection/tests/integration/targets/tower_project_manual/tasks/create_project_dir.yml +++ b/awx_collection/tests/integration/targets/tower_project_manual/tasks/create_project_dir.yml @@ -40,7 +40,7 @@ job_type: run module_args: "mkdir -p {{ project_base_dir }}/{{ project_dir_name }}" module_name: command - wait: True + wait: true always: - name: Delete path from setting From 4eb85ad23e5fe0c2355ba26ff3b3bb31031b1b0e Mon Sep 17 00:00:00 2001 From: Ryan Petrello Date: Fri, 26 Mar 2021 16:27:53 -0400 Subject: [PATCH 27/51] fix up a bug in rsyslogd error handling see: https://github.com/ansible/tower/issues/4915 --- awx/main/utils/handlers.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/awx/main/utils/handlers.py b/awx/main/utils/handlers.py index 19deb234b5..ef761159ed 100644 --- a/awx/main/utils/handlers.py +++ b/awx/main/utils/handlers.py @@ -5,6 +5,7 @@ import logging import sys import traceback +from datetime import datetime # Django from django.conf import settings @@ -34,7 +35,8 @@ class RSysLogHandler(logging.handlers.SysLogHandler): # because the alternative is blocking the # socket.send() in the Python process, which we definitely don't # want to do) - msg = f'{record.asctime} ERROR rsyslogd was unresponsive: ' + dt = datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S') + msg = f'{dt} ERROR rsyslogd was unresponsive: ' exc = traceback.format_exc() try: msg += exc.splitlines()[-1] From 6062d1ec9f27006603257ff0b01a7d3dbbec2874 Mon Sep 17 00:00:00 2001 From: Ryan Petrello Date: Fri, 26 Mar 2021 16:40:48 -0400 Subject: [PATCH 28/51] fix an HTTP 500 error for unauthenticated users see: https://github.com/ansible/awx/issues/7243 --- awx/api/views/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/awx/api/views/__init__.py b/awx/api/views/__init__.py index fa175eff7b..cf64e4114c 100644 --- a/awx/api/views/__init__.py +++ b/awx/api/views/__init__.py @@ -3043,6 +3043,8 @@ class WorkflowJobTemplateNodeCreateApproval(RetrieveAPIView): return Response(data, status=status.HTTP_201_CREATED) def check_permissions(self, request): + if not request.user.is_authenticated: + raise PermissionDenied() obj = self.get_object().workflow_job_template if request.method == 'POST': if not request.user.can_access(models.WorkflowJobTemplate, 'change', obj, request.data): From eac7c409d18b9ea2e4050081530d29fc446bcf20 Mon Sep 17 00:00:00 2001 From: Shane McDonald Date: Fri, 26 Mar 2021 16:52:19 -0400 Subject: [PATCH 29/51] Hide commands that are being run for `make black` --- Makefile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 18407ecb8f..e2cac573bd 100644 --- a/Makefile +++ b/Makefile @@ -272,12 +272,12 @@ reports: mkdir -p $@ black: reports - command -v black >/dev/null 2>&1 || { echo "could not find black on your PATH, you may need to \`pip install black\`, or set AWX_IGNORE_BLACK=1" && exit 1; } - (set -o pipefail && $@ $(BLACK_ARGS) awx awxkit awx_collection | tee reports/$@.report) + @command -v black >/dev/null 2>&1 || { echo "could not find black on your PATH, you may need to \`pip install black\`, or set AWX_IGNORE_BLACK=1" && exit 1; } + @(set -o pipefail && $@ $(BLACK_ARGS) awx awxkit awx_collection | tee reports/$@.report) .git/hooks/pre-commit: - echo "[ -z \$$AWX_IGNORE_BLACK ] && (black --check \`git diff --cached --name-only --diff-filter=AM | grep -E '\.py$\'\` || (echo 'To fix this, run \`make black\` to auto-format your code prior to commit, or set AWX_IGNORE_BLACK=1' && exit 1))" > .git/hooks/pre-commit - chmod +x .git/hooks/pre-commit + @echo "[ -z \$$AWX_IGNORE_BLACK ] && (black --check \`git diff --cached --name-only --diff-filter=AM | grep -E '\.py$\'\` || (echo 'To fix this, run \`make black\` to auto-format your code prior to commit, or set AWX_IGNORE_BLACK=1' && exit 1))" > .git/hooks/pre-commit + @chmod +x .git/hooks/pre-commit genschema: reports $(MAKE) swagger PYTEST_ARGS="--genschema --create-db " From dbcdbe0770641d2e6e0083d42a020b045f6520d8 Mon Sep 17 00:00:00 2001 From: sean-m-ssullivan Date: Sun, 28 Mar 2021 22:35:38 -0500 Subject: [PATCH 30/51] add logic to changed status --- awx_collection/plugins/modules/tower_project_update.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/awx_collection/plugins/modules/tower_project_update.py b/awx_collection/plugins/modules/tower_project_update.py index 0ee764a5e4..42be06ea12 100644 --- a/awx_collection/plugins/modules/tower_project_update.py +++ b/awx_collection/plugins/modules/tower_project_update.py @@ -34,6 +34,7 @@ options: wait: description: - Wait for the project to update. + - If scm revision has not changed module will return not changed. default: True type: bool interval: @@ -109,6 +110,9 @@ def main(): if project is None: module.fail_json(msg="Unable to find project") + if wait: + scm_revision_original = project['scm_revision'] + # Update the project result = module.post_endpoint(project['related']['update']) @@ -126,7 +130,10 @@ def main(): start = time.time() # Invoke wait function - module.wait_on_url(url=result['json']['url'], object_name=module.get_item_name(project), object_type='Project Update', timeout=timeout, interval=interval) + result = module.wait_on_url(url=result['json']['url'], object_name=module.get_item_name(project), object_type='Project Update', timeout=timeout, interval=interval) + scm_revision_new = result['json']['scm_revision'] + if scm_revision_new == scm_revision_original: + module.json_output['changed'] = False module.exit_json(**module.json_output) From c2b5ffcc1c2c2bc69e8dcb2abfee6020b9e26d39 Mon Sep 17 00:00:00 2001 From: sean-m-ssullivan Date: Mon, 29 Mar 2021 00:21:29 -0500 Subject: [PATCH 31/51] linting --- awx_collection/plugins/modules/tower_project_update.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/awx_collection/plugins/modules/tower_project_update.py b/awx_collection/plugins/modules/tower_project_update.py index 42be06ea12..796f910df7 100644 --- a/awx_collection/plugins/modules/tower_project_update.py +++ b/awx_collection/plugins/modules/tower_project_update.py @@ -130,7 +130,9 @@ def main(): start = time.time() # Invoke wait function - result = module.wait_on_url(url=result['json']['url'], object_name=module.get_item_name(project), object_type='Project Update', timeout=timeout, interval=interval) + result = module.wait_on_url( + url=result['json']['url'], object_name=module.get_item_name(project), object_type='Project Update', timeout=timeout, interval=interval + ) scm_revision_new = result['json']['scm_revision'] if scm_revision_new == scm_revision_original: module.json_output['changed'] = False From 8fb393c0a143bb078f5e6072916cabdaf7d9779c Mon Sep 17 00:00:00 2001 From: Shane McDonald Date: Mon, 29 Mar 2021 09:11:56 -0400 Subject: [PATCH 32/51] Fix awxkit function that detects private data directories in job args --- awxkit/awxkit/api/pages/unified_jobs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/awxkit/awxkit/api/pages/unified_jobs.py b/awxkit/awxkit/api/pages/unified_jobs.py index 09dea1ebbb..4f71c2eb6c 100644 --- a/awxkit/awxkit/api/pages/unified_jobs.py +++ b/awxkit/awxkit/api/pages/unified_jobs.py @@ -139,7 +139,7 @@ class UnifiedJob(HasStatus, base.Base): """ self.get() job_args = self.job_args - expected_prefix = '/tmp/awx_{}'.format(self.id) + expected_prefix = '/tmp/pdd_wrapper_{}'.format(self.id) for arg1, arg2 in zip(job_args[:-1], job_args[1:]): if arg1 == '-v': if ':' in arg2: From b6ccd02f3d7e83a3d1d7e8015dea211c9369016a Mon Sep 17 00:00:00 2001 From: Jeff Bradberry Date: Mon, 29 Mar 2021 10:39:22 -0400 Subject: [PATCH 33/51] Update the versioning on the docker-compose template Some versions of docker-compose will break with the new addition of name parameters without this. --- .../ansible/roles/sources/templates/docker-compose.yml.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/docker-compose/ansible/roles/sources/templates/docker-compose.yml.j2 b/tools/docker-compose/ansible/roles/sources/templates/docker-compose.yml.j2 index 3b32df378e..34d32df891 100644 --- a/tools/docker-compose/ansible/roles/sources/templates/docker-compose.yml.j2 +++ b/tools/docker-compose/ansible/roles/sources/templates/docker-compose.yml.j2 @@ -1,5 +1,5 @@ --- -version: '2' +version: '2.1' services: {% for i in range(cluster_node_count|int) %} {% set container_postfix = loop.index %} From 675286c1ac3f039ccff8e92a5be077ca4e39a8ff Mon Sep 17 00:00:00 2001 From: beeankha Date: Mon, 29 Mar 2021 11:52:20 -0400 Subject: [PATCH 34/51] Enable ?page_size=1 in URL to fetch correct objects on schedules endpoint --- .../0135_schedule_sort_fallback_to_id.py | 18 ++++++++++++++++++ awx/main/models/schedules.py | 2 +- 2 files changed, 19 insertions(+), 1 deletion(-) create mode 100644 awx/main/migrations/0135_schedule_sort_fallback_to_id.py diff --git a/awx/main/migrations/0135_schedule_sort_fallback_to_id.py b/awx/main/migrations/0135_schedule_sort_fallback_to_id.py new file mode 100644 index 0000000000..69969fafb4 --- /dev/null +++ b/awx/main/migrations/0135_schedule_sort_fallback_to_id.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.16 on 2021-03-29 15:30 + +from django.db import migrations +import django.db.models.expressions + + +class Migration(migrations.Migration): + + dependencies = [ + ('main', '0134_unifiedjob_ansible_version'), + ] + + operations = [ + migrations.AlterModelOptions( + name='schedule', + options={'ordering': [django.db.models.expressions.OrderBy(django.db.models.expressions.F('next_run'), descending=True, nulls_last=True), 'id']}, + ), + ] diff --git a/awx/main/models/schedules.py b/awx/main/models/schedules.py index d30d44372d..dca50d9232 100644 --- a/awx/main/models/schedules.py +++ b/awx/main/models/schedules.py @@ -63,7 +63,7 @@ class ScheduleManager(ScheduleFilterMethods, models.Manager): class Schedule(PrimordialModel, LaunchTimeConfig): class Meta: app_label = 'main' - ordering = ['-next_run'] + ordering = [models.F('next_run').desc(nulls_last=True), 'id'] unique_together = ('unified_job_template', 'name') objects = ScheduleManager() From 115a344842f9f6811ad096ec97c11f5cf6922317 Mon Sep 17 00:00:00 2001 From: nixocio Date: Mon, 29 Mar 2021 10:35:22 -0400 Subject: [PATCH 35/51] Add templates screen to EE Add templates screen to EE. See: https://github.com/ansible/awx/issues/9723 --- .../src/api/models/ExecutionEnvironments.js | 10 ++ awx/ui_next/src/api/models/Organizations.js | 6 +- .../ExecutionEnvironment.jsx | 11 ++ .../ExecutionEnvironmentTemplateList.jsx | 139 ++++++++++++++++++ .../ExecutionEnvironmentTemplateList.test.jsx | 116 +++++++++++++++ .../ExecutionEnvironmentTemplateListItem.jsx | 43 ++++++ ...cutionEnvironmentTemplateListItem.test.jsx | 48 ++++++ .../ExecutionEnvironmentTemplate/index.js | 1 + .../OrganizationExecEnvList.jsx | 2 +- 9 files changed, 371 insertions(+), 5 deletions(-) create mode 100644 awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentTemplate/ExecutionEnvironmentTemplateList.jsx create mode 100644 awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentTemplate/ExecutionEnvironmentTemplateList.test.jsx create mode 100644 awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentTemplate/ExecutionEnvironmentTemplateListItem.jsx create mode 100644 awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentTemplate/ExecutionEnvironmentTemplateListItem.test.jsx create mode 100644 awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentTemplate/index.js diff --git a/awx/ui_next/src/api/models/ExecutionEnvironments.js b/awx/ui_next/src/api/models/ExecutionEnvironments.js index 2df933d53a..ae3d128ed3 100644 --- a/awx/ui_next/src/api/models/ExecutionEnvironments.js +++ b/awx/ui_next/src/api/models/ExecutionEnvironments.js @@ -5,6 +5,16 @@ class ExecutionEnvironments extends Base { super(http); this.baseUrl = '/api/v2/execution_environments/'; } + + readUnifiedJobTemplates(id, params) { + return this.http.get(`${this.baseUrl}${id}/unified_job_templates/`, { + params, + }); + } + + readUnifiedJobTemplateOptions(id) { + return this.http.options(`${this.baseUrl}${id}/unified_job_templates/`); + } } export default ExecutionEnvironments; diff --git a/awx/ui_next/src/api/models/Organizations.js b/awx/ui_next/src/api/models/Organizations.js index fd980fece8..a2baa4f9c8 100644 --- a/awx/ui_next/src/api/models/Organizations.js +++ b/awx/ui_next/src/api/models/Organizations.js @@ -36,10 +36,8 @@ class Organizations extends InstanceGroupsMixin(NotificationsMixin(Base)) { }); } - readExecutionEnvironmentsOptions(id, params) { - return this.http.options(`${this.baseUrl}${id}/execution_environments/`, { - params, - }); + readExecutionEnvironmentsOptions(id) { + return this.http.options(`${this.baseUrl}${id}/execution_environments/`); } createUser(id, data) { diff --git a/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironment.jsx b/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironment.jsx index 55a3228e13..bb86dc2f57 100644 --- a/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironment.jsx +++ b/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironment.jsx @@ -20,6 +20,7 @@ import ContentLoading from '../../components/ContentLoading'; import ExecutionEnvironmentDetails from './ExecutionEnvironmentDetails'; import ExecutionEnvironmentEdit from './ExecutionEnvironmentEdit'; +import ExecutionEnvironmentTemplateList from './ExecutionEnvironmentTemplate'; function ExecutionEnvironment({ i18n, setBreadcrumb }) { const { id } = useParams(); @@ -64,6 +65,11 @@ function ExecutionEnvironment({ i18n, setBreadcrumb }) { link: `/execution_environments/${id}/details`, id: 0, }, + { + name: i18n._(t`Templates`), + link: `/execution_environments/${id}/templates`, + id: 1, + }, ]; if (!isLoading && contentError) { @@ -114,6 +120,11 @@ function ExecutionEnvironment({ i18n, setBreadcrumb }) { executionEnvironment={executionEnvironment} /> + + + )} diff --git a/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentTemplate/ExecutionEnvironmentTemplateList.jsx b/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentTemplate/ExecutionEnvironmentTemplateList.jsx new file mode 100644 index 0000000000..c2b36eee30 --- /dev/null +++ b/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentTemplate/ExecutionEnvironmentTemplateList.jsx @@ -0,0 +1,139 @@ +import React, { useEffect, useCallback } from 'react'; +import { useLocation } from 'react-router-dom'; +import { withI18n } from '@lingui/react'; +import { t } from '@lingui/macro'; +import { Card } from '@patternfly/react-core'; + +import { ExecutionEnvironmentsAPI } from '../../../api'; +import { getQSConfig, parseQueryString } from '../../../util/qs'; +import useRequest from '../../../util/useRequest'; +import DatalistToolbar from '../../../components/DataListToolbar'; +import PaginatedDataList from '../../../components/PaginatedDataList'; + +import ExecutionEnvironmentTemplateListItem from './ExecutionEnvironmentTemplateListItem'; + +const QS_CONFIG = getQSConfig( + 'execution_environments', + { + page: 1, + page_size: 20, + order_by: 'name', + type: 'job_template,workflow_job_template', + }, + ['id', 'page', 'page_size'] +); + +function ExecutionEnvironmentTemplateList({ i18n, executionEnvironment }) { + const { id } = executionEnvironment; + const location = useLocation(); + + const { + error: contentError, + isLoading, + request: fetchTemplates, + result: { + templates, + templatesCount, + relatedSearchableKeys, + searchableKeys, + }, + } = useRequest( + useCallback(async () => { + const params = parseQueryString(QS_CONFIG, location.search); + + const [response, responseActions] = await Promise.all([ + ExecutionEnvironmentsAPI.readUnifiedJobTemplates(id, params), + ExecutionEnvironmentsAPI.readUnifiedJobTemplateOptions(id), + ]); + + return { + templates: response.data.results, + templatesCount: response.data.count, + actions: responseActions.data.actions, + relatedSearchableKeys: ( + responseActions?.data?.related_search_fields || [] + ).map(val => val.slice(0, -8)), + searchableKeys: Object.keys( + responseActions.data.actions?.GET || {} + ).filter(key => responseActions.data.actions?.GET[key].filterable), + }; + }, [location, id]), + { + templates: [], + templatesCount: 0, + actions: {}, + relatedSearchableKeys: [], + searchableKeys: [], + } + ); + + useEffect(() => { + fetchTemplates(); + }, [fetchTemplates]); + + return ( + <> + + ( + + )} + renderItem={template => ( + + )} + /> + + + ); +} + +export default withI18n()(ExecutionEnvironmentTemplateList); diff --git a/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentTemplate/ExecutionEnvironmentTemplateList.test.jsx b/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentTemplate/ExecutionEnvironmentTemplateList.test.jsx new file mode 100644 index 0000000000..078d6d249d --- /dev/null +++ b/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentTemplate/ExecutionEnvironmentTemplateList.test.jsx @@ -0,0 +1,116 @@ +import React from 'react'; +import { act } from 'react-dom/test-utils'; + +import { + mountWithContexts, + waitForElement, +} from '../../../../testUtils/enzymeHelpers'; + +import { ExecutionEnvironmentsAPI } from '../../../api'; +import ExecutionEnvironmentTemplateList from './ExecutionEnvironmentTemplateList'; + +jest.mock('../../../api/'); + +const templates = { + data: { + count: 3, + results: [ + { + id: 1, + type: 'job_template', + name: 'Foo', + url: '/api/v2/job_templates/1/', + related: { + execution_environment: '/api/v2/execution_environments/1/', + }, + }, + { + id: 2, + type: 'workflow_job_template', + name: 'Bar', + url: '/api/v2/workflow_job_templates/2/', + related: { + execution_environment: '/api/v2/execution_environments/1/', + }, + }, + { + id: 3, + type: 'job_template', + name: 'Fuzz', + url: '/api/v2/job_templates/3/', + related: { + execution_environment: '/api/v2/execution_environments/1/', + }, + }, + ], + }, +}; + +const mockExecutionEnvironment = { + id: 1, + name: 'Default EE', +}; + +const options = { data: { actions: { GET: {} } } }; + +describe('', () => { + let wrapper; + + test('should mount successfully', async () => { + await act(async () => { + wrapper = mountWithContexts( + + ); + }); + await waitForElement( + wrapper, + 'ExecutionEnvironmentTemplateList', + el => el.length > 0 + ); + }); + + test('should have data fetched and render 3 rows', async () => { + ExecutionEnvironmentsAPI.readUnifiedJobTemplates.mockResolvedValue( + templates + ); + + ExecutionEnvironmentsAPI.readUnifiedJobTemplateOptions.mockResolvedValue( + options + ); + + await act(async () => { + wrapper = mountWithContexts( + + ); + }); + await waitForElement( + wrapper, + 'ExecutionEnvironmentTemplateList', + el => el.length > 0 + ); + + expect(wrapper.find('ExecutionEnvironmentTemplateListItem').length).toBe(3); + expect(ExecutionEnvironmentsAPI.readUnifiedJobTemplates).toBeCalled(); + expect(ExecutionEnvironmentsAPI.readUnifiedJobTemplateOptions).toBeCalled(); + }); + + test('should not render add button', async () => { + await act(async () => { + wrapper = mountWithContexts( + + ); + }); + waitForElement( + wrapper, + 'ExecutionEnvironmentTemplateList', + el => el.length > 0 + ); + expect(wrapper.find('ToolbarAddButton').length).toBe(0); + }); +}); diff --git a/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentTemplate/ExecutionEnvironmentTemplateListItem.jsx b/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentTemplate/ExecutionEnvironmentTemplateListItem.jsx new file mode 100644 index 0000000000..4a33126386 --- /dev/null +++ b/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentTemplate/ExecutionEnvironmentTemplateListItem.jsx @@ -0,0 +1,43 @@ +import React from 'react'; +import { withI18n } from '@lingui/react'; +import { t } from '@lingui/macro'; +import { Link } from 'react-router-dom'; +import { + DataListItem, + DataListItemRow, + DataListItemCells, +} from '@patternfly/react-core'; + +import DataListCell from '../../../components/DataListCell'; + +function ExecutionEnvironmentTemplateListItem({ template, detailUrl, i18n }) { + return ( + + + + + {template.name} + + , + + {template.type === 'job_template' + ? i18n._(t`Job Template`) + : i18n._(t`Workflow Job Template`)} + , + ]} + /> + + + ); +} + +export default withI18n()(ExecutionEnvironmentTemplateListItem); diff --git a/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentTemplate/ExecutionEnvironmentTemplateListItem.test.jsx b/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentTemplate/ExecutionEnvironmentTemplateListItem.test.jsx new file mode 100644 index 0000000000..9c107ab19b --- /dev/null +++ b/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentTemplate/ExecutionEnvironmentTemplateListItem.test.jsx @@ -0,0 +1,48 @@ +import React from 'react'; +import { act } from 'react-dom/test-utils'; + +import { mountWithContexts } from '../../../../testUtils/enzymeHelpers'; + +import ExecutionEnvironmentTemplateListItem from './ExecutionEnvironmentTemplateListItem'; + +describe('', () => { + let wrapper; + const template = { + id: 1, + name: 'Foo', + type: 'job_template', + }; + + test('should mount successfully', async () => { + await act(async () => { + wrapper = mountWithContexts( + + ); + }); + expect(wrapper.find('ExecutionEnvironmentTemplateListItem').length).toBe(1); + expect(wrapper.find('DataListCell[aria-label="Name"]').text()).toBe( + template.name + ); + expect( + wrapper.find('DataListCell[aria-label="Template type"]').text() + ).toBe('Job Template'); + }); + + test('should distinguish template types', async () => { + await act(async () => { + wrapper = mountWithContexts( + + ); + }); + expect(wrapper.find('ExecutionEnvironmentTemplateListItem').length).toBe(1); + expect( + wrapper.find('DataListCell[aria-label="Template type"]').text() + ).toBe('Workflow Job Template'); + }); +}); diff --git a/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentTemplate/index.js b/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentTemplate/index.js new file mode 100644 index 0000000000..3bdea254ee --- /dev/null +++ b/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentTemplate/index.js @@ -0,0 +1 @@ +export { default } from './ExecutionEnvironmentTemplateList'; diff --git a/awx/ui_next/src/screens/Organization/OrganizationExecEnvList/OrganizationExecEnvList.jsx b/awx/ui_next/src/screens/Organization/OrganizationExecEnvList/OrganizationExecEnvList.jsx index 9f2c4ae817..d567c24097 100644 --- a/awx/ui_next/src/screens/Organization/OrganizationExecEnvList/OrganizationExecEnvList.jsx +++ b/awx/ui_next/src/screens/Organization/OrganizationExecEnvList/OrganizationExecEnvList.jsx @@ -38,7 +38,7 @@ function OrganizationExecEnvList({ i18n, organization }) { const [response, responseActions] = await Promise.all([ OrganizationsAPI.readExecutionEnvironments(id, params), - OrganizationsAPI.readExecutionEnvironmentsOptions(id, params), + OrganizationsAPI.readExecutionEnvironmentsOptions(id), ]); return { From 1e9b22148668a0168ab5cb212a503beeea05d5fe Mon Sep 17 00:00:00 2001 From: Shane McDonald Date: Mon, 29 Mar 2021 13:05:57 -0400 Subject: [PATCH 36/51] Use revolved EE for Container Group tasks --- awx/main/tasks.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/awx/main/tasks.py b/awx/main/tasks.py index b5cb7ec1e8..6cadc1973e 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -3126,11 +3126,20 @@ class AWXReceptorJob: @property def pod_definition(self): + ee = self.task.instance.resolve_execution_environment() + default_pod_spec = { "apiVersion": "v1", "kind": "Pod", "metadata": {"namespace": settings.AWX_CONTAINER_GROUP_DEFAULT_NAMESPACE}, - "spec": {"containers": [{"image": settings.AWX_CONTAINER_GROUP_DEFAULT_IMAGE, "name": 'worker', "args": ['ansible-runner', 'worker']}]}, + "spec": { + "containers": [ + { + "image": ee.image, + "name": 'worker', + } + ], + }, } pod_spec_override = {} From eeb6aaaea955675304892001a88ecb2ca19c5893 Mon Sep 17 00:00:00 2001 From: Shane McDonald Date: Mon, 29 Mar 2021 13:08:43 -0400 Subject: [PATCH 37/51] Always use EE resolving logic --- awx/main/management/commands/create_preload_data.py | 6 +++--- awx/main/management/commands/inventory_import.py | 3 ++- awx/main/models/mixins.py | 10 ++-------- awx/main/utils/execution_environments.py | 9 +++++++++ awx/settings/defaults.py | 13 +++++++------ 5 files changed, 23 insertions(+), 18 deletions(-) create mode 100644 awx/main/utils/execution_environments.py diff --git a/awx/main/management/commands/create_preload_data.py b/awx/main/management/commands/create_preload_data.py index af5d8d9d9b..b40515321d 100644 --- a/awx/main/management/commands/create_preload_data.py +++ b/awx/main/management/commands/create_preload_data.py @@ -68,12 +68,12 @@ class Command(BaseCommand): print('Demo Credential, Inventory, and Job Template added.') changed = True - default_ee = settings.AWX_EXECUTION_ENVIRONMENT_DEFAULT_IMAGE - ee, created = ExecutionEnvironment.objects.get_or_create(name='Default EE', defaults={'image': default_ee, 'managed_by_tower': True}) + for ee in reversed(settings.DEFAULT_EXECUTION_ENVIRONMENTS): + _, created = ExecutionEnvironment.objects.get_or_create(name=ee['name'], defaults={'image': ee['image'], 'managed_by_tower': True}) if created: changed = True - print('Default Execution Environment registered.') + print('Default Execution Environment(s) registered.') if changed: print('(changed: True)') diff --git a/awx/main/management/commands/inventory_import.py b/awx/main/management/commands/inventory_import.py index 9cdd2f3017..af359128eb 100644 --- a/awx/main/management/commands/inventory_import.py +++ b/awx/main/management/commands/inventory_import.py @@ -29,6 +29,7 @@ from awx.main.utils.safe_yaml import sanitize_jinja # other AWX imports from awx.main.models.rbac import batch_role_ancestor_rebuilding from awx.main.utils import ignore_inventory_computed_fields, get_licenser +from awx.main.utils.execution_environments import get_execution_environment_default from awx.main.signals import disable_activity_stream from awx.main.constants import STANDARD_INVENTORY_UPDATE_ENV from awx.main.utils.pglock import advisory_lock @@ -90,7 +91,7 @@ class AnsibleInventoryLoader(object): bargs.extend(['-v', '{0}:{0}:Z'.format(self.source)]) for key, value in STANDARD_INVENTORY_UPDATE_ENV.items(): bargs.extend(['-e', '{0}={1}'.format(key, value)]) - bargs.extend([settings.AWX_EXECUTION_ENVIRONMENT_DEFAULT_IMAGE]) + bargs.extend([get_execution_environment_default().image]) bargs.extend(['ansible-inventory', '-i', self.source]) bargs.extend(['--playbook-dir', functioning_dir(self.source)]) if self.verbosity: diff --git a/awx/main/models/mixins.py b/awx/main/models/mixins.py index 8055502096..645d0ebe09 100644 --- a/awx/main/models/mixins.py +++ b/awx/main/models/mixins.py @@ -21,6 +21,7 @@ from django.utils.translation import ugettext_lazy as _ from awx.main.models.base import prevent_search from awx.main.models.rbac import Role, RoleAncestorEntry, get_roles_on_resource from awx.main.utils import parse_yaml_or_json, get_custom_venv_choices, get_licenser, polymorphic +from awx.main.utils.execution_environments import get_execution_environment_default from awx.main.utils.encryption import decrypt_value, get_encryption_key, is_encrypted from awx.main.utils.polymorphic import build_polymorphic_ctypes_map from awx.main.fields import JSONField, AskForField @@ -461,13 +462,6 @@ class ExecutionEnvironmentMixin(models.Model): help_text=_('The container image to be used for execution.'), ) - def get_execution_environment_default(self): - from awx.main.models.execution_environments import ExecutionEnvironment - - if settings.DEFAULT_EXECUTION_ENVIRONMENT is not None: - return settings.DEFAULT_EXECUTION_ENVIRONMENT - return ExecutionEnvironment.objects.filter(organization=None, managed_by_tower=True).first() - def resolve_execution_environment(self): """ Return the execution environment that should be used when creating a new job. @@ -482,7 +476,7 @@ class ExecutionEnvironmentMixin(models.Model): if self.inventory.organization.default_environment is not None: return self.inventory.organization.default_environment - return self.get_execution_environment_default() + return get_execution_environment_default() class CustomVirtualEnvMixin(models.Model): diff --git a/awx/main/utils/execution_environments.py b/awx/main/utils/execution_environments.py new file mode 100644 index 0000000000..d705f93210 --- /dev/null +++ b/awx/main/utils/execution_environments.py @@ -0,0 +1,9 @@ +from django.conf import settings + +from awx.main.models.execution_environments import ExecutionEnvironment + + +def get_execution_environment_default(): + if settings.DEFAULT_EXECUTION_ENVIRONMENT is not None: + return settings.DEFAULT_EXECUTION_ENVIRONMENT + return ExecutionEnvironment.objects.filter(organization=None, managed_by_tower=True).first() diff --git a/awx/settings/defaults.py b/awx/settings/defaults.py index 2daa33d4b3..11208e82e0 100644 --- a/awx/settings/defaults.py +++ b/awx/settings/defaults.py @@ -68,17 +68,11 @@ DATABASES = { # the K8S cluster where awx itself is running) IS_K8S = False -# TODO: remove this setting in favor of a default execution environment -AWX_EXECUTION_ENVIRONMENT_DEFAULT_IMAGE = 'quay.io/ansible/awx-ee' - AWX_CONTAINER_GROUP_K8S_API_TIMEOUT = 10 AWX_CONTAINER_GROUP_POD_LAUNCH_RETRIES = 100 AWX_CONTAINER_GROUP_POD_LAUNCH_RETRY_DELAY = 5 AWX_CONTAINER_GROUP_DEFAULT_NAMESPACE = os.getenv('MY_POD_NAMESPACE', 'default') -# TODO: remove this setting in favor of a default execution environment -AWX_CONTAINER_GROUP_DEFAULT_IMAGE = AWX_EXECUTION_ENVIRONMENT_DEFAULT_IMAGE - # Internationalization # https://docs.djangoproject.com/en/dev/topics/i18n/ # @@ -182,8 +176,15 @@ REMOTE_HOST_HEADERS = ['REMOTE_ADDR', 'REMOTE_HOST'] PROXY_IP_ALLOWED_LIST = [] CUSTOM_VENV_PATHS = [] + +# Warning: this is a placeholder for a configure tower-in-tower setting +# This should not be set via a file. DEFAULT_EXECUTION_ENVIRONMENT = None +# This list is used for creating default EEs when running awx-manage create_preload_data. +# Should be ordered from highest to lowest precedence. +DEFAULT_EXECUTION_ENVIRONMENTS = [{'name': 'AWX EE 0.1.1', 'image': 'quay.io/ansible/awx-ee:0.1.1'}] + # Note: This setting may be overridden by database settings. STDOUT_MAX_BYTES_DISPLAY = 1048576 From 9d17d40b86810b69ae6dd058db6f1e6bd24796df Mon Sep 17 00:00:00 2001 From: Shane McDonald Date: Mon, 29 Mar 2021 13:09:31 -0400 Subject: [PATCH 38/51] Add setting for keeping container group pod after job run Helpful for debugging --- awx/main/tasks.py | 2 +- awx/settings/defaults.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/awx/main/tasks.py b/awx/main/tasks.py index 6cadc1973e..43a629b288 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -3010,7 +3010,7 @@ class AWXReceptorJob: return self._run_internal(receptor_ctl) finally: # Make sure to always release the work unit if we established it - if self.unit_id is not None: + if self.unit_id is not None and not settings.AWX_CONTAINER_GROUP_KEEP_POD: receptor_ctl.simple_command(f"work release {self.unit_id}") def _run_internal(self, receptor_ctl): diff --git a/awx/settings/defaults.py b/awx/settings/defaults.py index 11208e82e0..28f5b20ec8 100644 --- a/awx/settings/defaults.py +++ b/awx/settings/defaults.py @@ -68,6 +68,7 @@ DATABASES = { # the K8S cluster where awx itself is running) IS_K8S = False +AWX_CONTAINER_GROUP_KEEP_POD = True AWX_CONTAINER_GROUP_K8S_API_TIMEOUT = 10 AWX_CONTAINER_GROUP_POD_LAUNCH_RETRIES = 100 AWX_CONTAINER_GROUP_POD_LAUNCH_RETRY_DELAY = 5 From 9bc0bf0ee728d95a715f2a6db9ccd5c68961010e Mon Sep 17 00:00:00 2001 From: Shane McDonald Date: Mon, 29 Mar 2021 13:09:56 -0400 Subject: [PATCH 39/51] Fix inventory updates when running inside of k8s --- awx/main/models/inventory.py | 4 ++++ awx/main/tasks.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/awx/main/models/inventory.py b/awx/main/models/inventory.py index cbec2963ca..6fabdf7567 100644 --- a/awx/main/models/inventory.py +++ b/awx/main/models/inventory.py @@ -1227,6 +1227,10 @@ class InventoryUpdate(UnifiedJob, InventorySourceOptions, JobNotificationMixin, null=True, ) + @property + def is_container_group_task(self): + return bool(self.instance_group and self.instance_group.is_container_group) + def _get_parent_field_name(self): return 'inventory_source' diff --git a/awx/main/tasks.py b/awx/main/tasks.py index 43a629b288..e05045574f 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -2505,7 +2505,7 @@ class RunInventoryUpdate(BaseTask): args.append(container_location) args.append('--output') - args.append(os.path.join('/runner', 'artifacts', 'output.json')) + args.append(os.path.join('/runner', 'artifacts', str(inventory_update.id), 'output.json')) if os.path.isdir(source_location): playbook_dir = container_location From 91351f7e3bfaf6343db543b470fa8366b23c53e0 Mon Sep 17 00:00:00 2001 From: Shane McDonald Date: Mon, 29 Mar 2021 13:13:05 -0400 Subject: [PATCH 40/51] Fix default setting for KEEP_POD --- awx/settings/defaults.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/awx/settings/defaults.py b/awx/settings/defaults.py index 28f5b20ec8..b6a3966647 100644 --- a/awx/settings/defaults.py +++ b/awx/settings/defaults.py @@ -68,7 +68,7 @@ DATABASES = { # the K8S cluster where awx itself is running) IS_K8S = False -AWX_CONTAINER_GROUP_KEEP_POD = True +AWX_CONTAINER_GROUP_KEEP_POD = False AWX_CONTAINER_GROUP_K8S_API_TIMEOUT = 10 AWX_CONTAINER_GROUP_POD_LAUNCH_RETRIES = 100 AWX_CONTAINER_GROUP_POD_LAUNCH_RETRY_DELAY = 5 From 3d533e566145d0f2774e7297737564236cae01c7 Mon Sep 17 00:00:00 2001 From: Shane McDonald Date: Mon, 29 Mar 2021 13:26:59 -0400 Subject: [PATCH 41/51] Fix container group OPTIONS request --- awx/main/tasks.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/awx/main/tasks.py b/awx/main/tasks.py index e05045574f..221a9ce600 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -97,6 +97,7 @@ from awx.main.utils import ( deepmerge, parse_yaml_or_json, ) +from awx.main.utils.execution_environments import get_execution_environment_default from awx.main.utils.ansible import read_ansible_config from awx.main.utils.external_logging import reconfigure_rsyslog from awx.main.utils.safe_yaml import safe_dump, sanitize_jinja @@ -3126,7 +3127,10 @@ class AWXReceptorJob: @property def pod_definition(self): - ee = self.task.instance.resolve_execution_environment() + if self.task: + ee = self.task.instance.resolve_execution_environment() + else: + ee = get_execution_environment_default() default_pod_spec = { "apiVersion": "v1", From f80c2cbfc3af84fe8f23534476e9d3ffc10d09a0 Mon Sep 17 00:00:00 2001 From: Shane McDonald Date: Mon, 29 Mar 2021 14:45:03 -0400 Subject: [PATCH 42/51] Delete test for unused code --- .../tests/unit/scheduler/test_kubernetes.py | 49 ------------------- 1 file changed, 49 deletions(-) delete mode 100644 awx/main/tests/unit/scheduler/test_kubernetes.py diff --git a/awx/main/tests/unit/scheduler/test_kubernetes.py b/awx/main/tests/unit/scheduler/test_kubernetes.py deleted file mode 100644 index 1f51401fe4..0000000000 --- a/awx/main/tests/unit/scheduler/test_kubernetes.py +++ /dev/null @@ -1,49 +0,0 @@ -import pytest -from django.conf import settings - -from awx.main.models import ( - InstanceGroup, - Job, - JobTemplate, - Project, - Inventory, -) -from awx.main.scheduler.kubernetes import PodManager - - -@pytest.fixture -def container_group(): - instance_group = InstanceGroup(name='container-group', id=1) - - return instance_group - - -@pytest.fixture -def job(container_group): - return Job(pk=1, id=1, project=Project(), instance_group=container_group, inventory=Inventory(), job_template=JobTemplate(id=1, name='foo')) - - -def test_default_pod_spec(job): - default_image = PodManager(job).pod_definition['spec']['containers'][0]['image'] - assert default_image == settings.AWX_CONTAINER_GROUP_DEFAULT_IMAGE - - -def test_custom_pod_spec(job): - job.instance_group.pod_spec_override = """ - spec: - containers: - - image: my-custom-image - """ - custom_image = PodManager(job).pod_definition['spec']['containers'][0]['image'] - assert custom_image == 'my-custom-image' - - -def test_pod_manager_namespace_property(job): - pm = PodManager(job) - assert pm.namespace == settings.AWX_CONTAINER_GROUP_DEFAULT_NAMESPACE - - job.instance_group.pod_spec_override = """ - metadata: - namespace: my-namespace - """ - assert PodManager(job).namespace == 'my-namespace' From a5b29201a4bb5f6141446298b969b6b60d103051 Mon Sep 17 00:00:00 2001 From: Shane McDonald Date: Mon, 29 Mar 2021 14:45:21 -0400 Subject: [PATCH 43/51] Update tests to use ee fixture --- awx/main/tests/functional/api/test_instance_group.py | 2 +- awx/main/tests/functional/conftest.py | 4 ++-- .../task_management/test_container_groups.py | 11 +++++++---- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/awx/main/tests/functional/api/test_instance_group.py b/awx/main/tests/functional/api/test_instance_group.py index 967775dd74..c3cf44fd74 100644 --- a/awx/main/tests/functional/api/test_instance_group.py +++ b/awx/main/tests/functional/api/test_instance_group.py @@ -140,7 +140,7 @@ def test_delete_instance_group_jobs_running(delete, instance_group_jobs_running, @pytest.mark.django_db -def test_delete_rename_tower_instance_group_prevented(delete, options, tower_instance_group, instance_group, user, patch): +def test_delete_rename_tower_instance_group_prevented(delete, options, tower_instance_group, instance_group, user, patch, execution_environment): url = reverse("api:instance_group_detail", kwargs={'pk': tower_instance_group.pk}) super_user = user('bob', True) diff --git a/awx/main/tests/functional/conftest.py b/awx/main/tests/functional/conftest.py index 96101ffb41..c54b06b86f 100644 --- a/awx/main/tests/functional/conftest.py +++ b/awx/main/tests/functional/conftest.py @@ -829,5 +829,5 @@ def slice_job_factory(slice_jt_factory): @pytest.fixture -def execution_environment(organization): - return ExecutionEnvironment.objects.create(name="test-ee", description="test-ee", organization=organization) +def execution_environment(): + return ExecutionEnvironment.objects.create(name="test-ee", description="test-ee", managed_by_tower=True) diff --git a/awx/main/tests/functional/task_management/test_container_groups.py b/awx/main/tests/functional/task_management/test_container_groups.py index e88ef2deb1..7bbdac218d 100644 --- a/awx/main/tests/functional/task_management/test_container_groups.py +++ b/awx/main/tests/functional/task_management/test_container_groups.py @@ -1,10 +1,11 @@ import subprocess import base64 +from collections import namedtuple from unittest import mock # noqa import pytest -from awx.main.scheduler.kubernetes import PodManager +from awx.main.tasks import AWXReceptorJob from awx.main.utils import ( create_temporary_fifo, ) @@ -34,7 +35,7 @@ def test_containerized_job(containerized_job): @pytest.mark.django_db -def test_kubectl_ssl_verification(containerized_job): +def test_kubectl_ssl_verification(containerized_job, execution_environment): cred = containerized_job.instance_group.credential cred.inputs['verify_ssl'] = True key_material = subprocess.run('openssl genrsa 2> /dev/null', shell=True, check=True, stdout=subprocess.PIPE) @@ -46,6 +47,8 @@ def test_kubectl_ssl_verification(containerized_job): cert = subprocess.run(cmd.strip(), shell=True, check=True, stdout=subprocess.PIPE) cred.inputs['ssl_ca_cert'] = cert.stdout cred.save() - pm = PodManager(containerized_job) - ca_data = pm.kube_config['clusters'][0]['cluster']['certificate-authority-data'] + RunJob = namedtuple('RunJob', ['instance', 'build_execution_environment_params']) + rj = RunJob(instance=containerized_job, build_execution_environment_params=lambda x: {}) + receptor_job = AWXReceptorJob(rj, runner_params={'settings': {}}) + ca_data = receptor_job.kube_config['clusters'][0]['cluster']['certificate-authority-data'] assert cert.stdout == base64.b64decode(ca_data.encode()) From b73759e3802ee36a5dac6e07c84f94f301551de9 Mon Sep 17 00:00:00 2001 From: Shane McDonald Date: Mon, 29 Mar 2021 15:03:28 -0400 Subject: [PATCH 44/51] Fix collection test --- awx_collection/test/awx/conftest.py | 7 ++++++- awx_collection/test/awx/test_completeness.py | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/awx_collection/test/awx/conftest.py b/awx_collection/test/awx/conftest.py index 969bb96da0..4d09cb5930 100644 --- a/awx_collection/test/awx/conftest.py +++ b/awx_collection/test/awx/conftest.py @@ -16,7 +16,7 @@ from requests.models import Response, PreparedRequest import pytest from awx.main.tests.functional.conftest import _request -from awx.main.models import Organization, Project, Inventory, JobTemplate, Credential, CredentialType +from awx.main.models import Organization, Project, Inventory, JobTemplate, Credential, CredentialType, ExecutionEnvironment from django.db import transaction @@ -261,3 +261,8 @@ def silence_warning(): """Warnings use global variable, same as deprecations.""" with mock.patch('ansible.module_utils.basic.AnsibleModule.warn') as this_mock: yield this_mock + + +@pytest.fixture +def execution_environment(): + return ExecutionEnvironment.objects.create(name="test-ee", description="test-ee", managed_by_tower=True) diff --git a/awx_collection/test/awx/test_completeness.py b/awx_collection/test/awx/test_completeness.py index 9deced2485..d639f828cd 100644 --- a/awx_collection/test/awx/test_completeness.py +++ b/awx_collection/test/awx/test_completeness.py @@ -157,7 +157,7 @@ def determine_state(module_id, endpoint, module, parameter, api_option, module_o return 'OK' -def test_completeness(collection_import, request, admin_user, job_template): +def test_completeness(collection_import, request, admin_user, job_template, execution_environment): option_comparison = {} # Load a list of existing module files from disk base_folder = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)) From ac41af8a5462d87f144ced0c402b9c57cb619deb Mon Sep 17 00:00:00 2001 From: nixocio Date: Wed, 24 Mar 2021 15:00:35 -0400 Subject: [PATCH 45/51] Add managed by tower as part of the EE details page Add managed by tower as part of the EE details page. See: https://github.com/ansible/awx/issues/8171 --- .../ExecutionEnvironmentDetails.jsx | 6 ++++++ .../ExecutionEnvironmentDetails.test.jsx | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentDetails/ExecutionEnvironmentDetails.jsx b/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentDetails/ExecutionEnvironmentDetails.jsx index a5c89f8d89..34de23c119 100644 --- a/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentDetails/ExecutionEnvironmentDetails.jsx +++ b/awx/ui_next/src/screens/ExecutionEnvironment/ExecutionEnvironmentDetails/ExecutionEnvironmentDetails.jsx @@ -64,6 +64,11 @@ function ExecutionEnvironmentDetails({ executionEnvironment, i18n }) { value={description} dataCy="execution-environment-detail-description" /> + + ', () => { expect( wrapper.find('Detail[label="Credential"]').prop('value').props.children ).toEqual(executionEnvironment.summary_fields.credential.name); + expect( + wrapper.find('Detail[label="Managed by Tower"]').prop('value') + ).toEqual('False'); const dates = wrapper.find('UserDateDetail'); expect(dates).toHaveLength(2); expect(dates.at(0).prop('date')).toEqual(executionEnvironment.created); @@ -167,6 +170,9 @@ describe('', () => { expect( wrapper.find('Detail[label="Credential"]').prop('value').props.children ).toEqual(executionEnvironment.summary_fields.credential.name); + expect( + wrapper.find('Detail[label="Managed by Tower"]').prop('value') + ).toEqual('True'); const dates = wrapper.find('UserDateDetail'); expect(dates).toHaveLength(2); expect(dates.at(0).prop('date')).toEqual(executionEnvironment.created); From b1119d2972d3a10db6b9f16c04785f7af3212c9c Mon Sep 17 00:00:00 2001 From: beeankha Date: Mon, 29 Mar 2021 16:11:06 -0400 Subject: [PATCH 46/51] Fix format specification linting erros --- awx_collection/plugins/module_utils/tower_api.py | 4 ++-- awx_collection/test/awx/test_module_utils.py | 8 +++++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/awx_collection/plugins/module_utils/tower_api.py b/awx_collection/plugins/module_utils/tower_api.py index f6c63b08de..c637f0f7b7 100644 --- a/awx_collection/plugins/module_utils/tower_api.py +++ b/awx_collection/plugins/module_utils/tower_api.py @@ -266,8 +266,8 @@ class TowerAPIModule(TowerModule): collection_compare_ver = parsed_collection_version[0] tower_compare_ver = parsed_tower_version[0] else: - collection_compare_ver = "{}.{}".format(parsed_collection_version[0], parsed_collection_version[1]) - tower_compare_ver = '{}.{}'.format(parsed_tower_version[0], parsed_tower_version[1]) + collection_compare_ver = "{0}.{1}".format(parsed_collection_version[0], parsed_collection_version[1]) + tower_compare_ver = '{0}.{1}'.format(parsed_tower_version[0], parsed_tower_version[1]) if self._COLLECTION_TYPE not in self.collection_to_version or self.collection_to_version[self._COLLECTION_TYPE] != tower_type: self.warn("You are using the {0} version of this collection but connecting to {1}".format(self._COLLECTION_TYPE, tower_type)) diff --git a/awx_collection/test/awx/test_module_utils.py b/awx_collection/test/awx/test_module_utils.py index 89bd44154e..a215db35fc 100644 --- a/awx_collection/test/awx/test_module_utils.py +++ b/awx_collection/test/awx/test_module_utils.py @@ -59,7 +59,7 @@ def test_version_warning(collection_import, silence_warning): my_module._COLLECTION_TYPE = "awx" my_module.get_endpoint('ping') silence_warning.assert_called_once_with( - 'You are running collection version {} but connecting to {} version {}'.format(my_module._COLLECTION_VERSION, awx_name, ping_version) + 'You are running collection version {0} but connecting to {1} version {2}'.format(my_module._COLLECTION_VERSION, awx_name, ping_version) ) @@ -107,7 +107,7 @@ def test_version_warning_strictness_tower(collection_import, silence_warning): my_module._COLLECTION_TYPE = "tower" my_module.get_endpoint('ping') silence_warning.assert_called_once_with( - 'You are running collection version {} but connecting to {} version {}'.format(my_module._COLLECTION_VERSION, tower_name, ping_version) + 'You are running collection version {0} but connecting to {1} version {2}'.format(my_module._COLLECTION_VERSION, tower_name, ping_version) ) @@ -121,7 +121,9 @@ def test_type_warning(collection_import, silence_warning): my_module._COLLECTION_VERSION = ping_version my_module._COLLECTION_TYPE = "tower" my_module.get_endpoint('ping') - silence_warning.assert_called_once_with('You are using the {} version of this collection but connecting to {}'.format(my_module._COLLECTION_TYPE, awx_name)) + silence_warning.assert_called_once_with( + 'You are using the {0} version of this collection but connecting to {1}'.format(my_module._COLLECTION_TYPE, awx_name) + ) def test_duplicate_config(collection_import, silence_warning): From 4beeeae9f1bef82aacb6152e84692eb55af1e235 Mon Sep 17 00:00:00 2001 From: Shane McDonald Date: Mon, 29 Mar 2021 17:33:40 -0400 Subject: [PATCH 47/51] Fix k8s credentials that use a custom ca cert --- awx/main/models/credential/injectors.py | 2 +- awx/main/tests/unit/test_tasks.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/awx/main/models/credential/injectors.py b/awx/main/models/credential/injectors.py index b5f7e37fed..246ab0d4e4 100644 --- a/awx/main/models/credential/injectors.py +++ b/awx/main/models/credential/injectors.py @@ -115,6 +115,6 @@ def kubernetes_bearer_token(cred, env, private_data_dir): with os.fdopen(handle, 'w') as f: os.chmod(path, stat.S_IRUSR | stat.S_IWUSR) f.write(cred.get_input('ssl_ca_cert')) - env['K8S_AUTH_SSL_CA_CERT'] = path + env['K8S_AUTH_SSL_CA_CERT'] = os.path.join('/runner', os.path.basename(path)) else: env['K8S_AUTH_VERIFY_SSL'] = 'False' diff --git a/awx/main/tests/unit/test_tasks.py b/awx/main/tests/unit/test_tasks.py index 26df22c4f2..5d600548a3 100644 --- a/awx/main/tests/unit/test_tasks.py +++ b/awx/main/tests/unit/test_tasks.py @@ -1003,7 +1003,8 @@ class TestJobCredentials(TestJobExecution): if verify: assert env['K8S_AUTH_VERIFY_SSL'] == 'True' - cert = open(env['K8S_AUTH_SSL_CA_CERT'], 'r').read() + local_path = os.path.join(private_data_dir, os.path.basename(env['K8S_AUTH_SSL_CA_CERT'])) + cert = open(local_path, 'r').read() assert cert == 'CERTDATA' else: assert env['K8S_AUTH_VERIFY_SSL'] == 'False' From fce544bb73d79f643dabb69352188edd5480f323 Mon Sep 17 00:00:00 2001 From: sean-m-ssullivan Date: Mon, 29 Mar 2021 22:14:06 -0500 Subject: [PATCH 48/51] add test logic --- .../integration/targets/tower_project_update/tasks/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/awx_collection/tests/integration/targets/tower_project_update/tasks/main.yml b/awx_collection/tests/integration/targets/tower_project_update/tasks/main.yml index 08b9852018..dd614d552a 100644 --- a/awx_collection/tests/integration/targets/tower_project_update/tasks/main.yml +++ b/awx_collection/tests/integration/targets/tower_project_update/tasks/main.yml @@ -53,6 +53,7 @@ - assert: that: - result is successful + - result is not changed - name: Delete the test project 1 tower_project: From 54308c5fa15b62373c5ee562efe7f634ed3ff94e Mon Sep 17 00:00:00 2001 From: Shane McDonald Date: Tue, 30 Mar 2021 09:08:39 -0400 Subject: [PATCH 49/51] Use Ansible Runner 2.0 alpha 1 --- requirements/requirements.in | 2 +- requirements/requirements.txt | 3 +-- requirements/requirements_git.txt | 1 - 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/requirements/requirements.in b/requirements/requirements.in index 1970b215fb..50ee011690 100644 --- a/requirements/requirements.in +++ b/requirements/requirements.in @@ -1,5 +1,5 @@ aiohttp -ansible-runner>=1.4.7 +ansible-runner==2.0.0a1 ansiconv==1.0.0 # UPGRADE BLOCKER: from 2013, consider replacing instead of upgrading asciichartpy autobahn>=20.12.3 # CVE-2020-35678 diff --git a/requirements/requirements.txt b/requirements/requirements.txt index 9450d4f879..0ebb97cb02 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -4,10 +4,9 @@ aiohttp==3.6.2 # via -r /awx_devel/requirements/requirements.in aioredis==1.3.1 # via channels-redis -#ansible-runner==1.4.7 +ansible-runner==2.0.0a1 # via # -r /awx_devel/requirements/requirements.in - # -r /awx_devel/requirements/requirements_git.txt ansiconv==1.0.0 # via -r /awx_devel/requirements/requirements.in asciichartpy==1.5.25 diff --git a/requirements/requirements_git.txt b/requirements/requirements_git.txt index f2f3abaa7a..be9973c1f8 100644 --- a/requirements/requirements_git.txt +++ b/requirements/requirements_git.txt @@ -1,3 +1,2 @@ git+https://github.com/ansible/system-certifi.git@devel#egg=certifi -git+git://github.com/ansible/ansible-runner@devel#egg=ansible-runner git+https://github.com/project-receptor/receptor.git@0.9.6#egg=receptorctl&subdirectory=receptorctl From dde408ea1a275171fbb4098ddfa78af387314cfa Mon Sep 17 00:00:00 2001 From: sean-m-ssullivan Date: Tue, 30 Mar 2021 09:35:52 -0500 Subject: [PATCH 50/51] update docs --- awx_collection/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/awx_collection/README.md b/awx_collection/README.md index 61480a1b3d..506d00e4a3 100644 --- a/awx_collection/README.md +++ b/awx_collection/README.md @@ -72,6 +72,8 @@ Notable releases of the `awx.awx` collection: The following notes are changes that may require changes to playbooks: - When a project is created, it will wait for the update/sync to finish by default; this can be turned off with the `wait` parameter, if desired. + - When using the wait parameter with project update, if the project did not undergo a revision update, the result will be + 'not changed' - Creating a "scan" type job template is no longer supported. - Specifying a custom certificate via the `TOWER_CERTIFICATE` environment variable no longer works. - Type changes of variable fields: From 4a726b7f6f2ca496e4b5e80df7551097d81bacf3 Mon Sep 17 00:00:00 2001 From: Shane McDonald Date: Tue, 30 Mar 2021 13:33:17 -0400 Subject: [PATCH 51/51] Fix race condition that causes InvalidGitRepositoryError --- awx/main/tasks.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/awx/main/tasks.py b/awx/main/tasks.py index 221a9ce600..992620fbd8 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -1807,13 +1807,14 @@ class RunJob(BaseTask): logger.debug('Performing fresh clone of {} on this instance.'.format(job.project)) sync_needs.append(source_update_tag) elif job.project.scm_type == 'git' and job.project.scm_revision and (not branch_override): - git_repo = git.Repo(project_path) try: + git_repo = git.Repo(project_path) + if job_revision == git_repo.head.commit.hexsha: logger.debug('Skipping project sync for {} because commit is locally available'.format(job.log_format)) else: sync_needs.append(source_update_tag) - except (ValueError, BadGitName): + except (ValueError, BadGitName, git.exc.InvalidGitRepositoryError): logger.debug('Needed commit for {} not in local source tree, will sync with remote'.format(job.log_format)) sync_needs.append(source_update_tag) else: