From 216454d29880161a466427d512ff35b3d16f7b9d Mon Sep 17 00:00:00 2001 From: chris meyers Date: Mon, 11 May 2020 10:45:53 -0400 Subject: [PATCH 1/6] cleanup channel groups on start * There are 2 data-structures that django channels redis uses: (1) zset and (2) list. (1) is used for group membership where the key is the logic user group and the value(s) are websocket clients. The score of the zset entry is used for group expiration. We can not rely on group expiration for clean-up because there is no interface privided by redis channels to refresh the expiration. Choosing a small value for group_expiry could result on our websocket backplane group expiring, which would result in job events not being delivered. Instead, we increase the group expiration to 5 years and clean up on daphne service start. * The list (2) data-structure is used by django channels redis to queue websocket events per-websocket-client as needed. The need arises to queue per-websocket-client events when the consumer can not keep up with the producer. The consumer here is daphne, the producer is AWX. * When AWX is operating healthy group membership in Redis is reflective of the real-world. When AWX is unhealthy i.e. daphne cycles, the zset will contain stale websocket client entries. This can be observed by running `zrange asgi::group:jobs-status_changed 0 -1`. If the entries returned look like: specific.fUkXXpYj!DKOIfwPICNgw specific.fUkXXpYj!FQcdopZeiRdG specific.lpTSAgnk!IOKldfzcfdDp specific.lpTSAgnk!NbvRUZsDpIQx The entries with `fUkXXpYj` are stale. Note that this changeset fixes this by removing all `asgi:*` entries on daphne start. * Also note that individual message themselves have an expiration that is configurable and defaults to 60. * Also note that zset's tracking group membership will be deleted by django channels redis when they are empty. --- awx/main/routing.py | 25 ++++++++++++++++++++++++- awx/settings/defaults.py | 1 + 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/awx/main/routing.py b/awx/main/routing.py index 090634cbb8..2866d46ed0 100644 --- a/awx/main/routing.py +++ b/awx/main/routing.py @@ -1,14 +1,37 @@ +import redis +import logging + from django.conf.urls import url +from django.conf import settings + from channels.auth import AuthMiddlewareStack from channels.routing import ProtocolTypeRouter, URLRouter + from . import consumers + +logger = logging.getLogger('awx.main.routing') + + +class AWXProtocolTypeRouter(ProtocolTypeRouter): + def __init__(self, *args, **kwargs): + try: + r = redis.Redis.from_url(settings.BROKER_URL) + for k in r.scan_iter('asgi:*', 500): + logger.debug(f"cleaning up Redis key {k}") + r.delete(k) + except redis.exceptions.RedisError as e: + logger.warn("encountered an error communicating with redis.") + raise e + super().__init__(*args, **kwargs) + + websocket_urlpatterns = [ url(r'websocket/$', consumers.EventConsumer), url(r'websocket/broadcast/$', consumers.BroadcastConsumer), ] -application = ProtocolTypeRouter({ +application = AWXProtocolTypeRouter({ 'websocket': AuthMiddlewareStack( URLRouter(websocket_urlpatterns) ), diff --git a/awx/settings/defaults.py b/awx/settings/defaults.py index 25acabda7b..031945dd6d 100644 --- a/awx/settings/defaults.py +++ b/awx/settings/defaults.py @@ -957,6 +957,7 @@ CHANNEL_LAYERS = { "CONFIG": { "hosts": [BROKER_URL], "capacity": 10000, + "group_expiry": 157784760, # 5 years }, }, } From 345f1db994f28fc0a962172b1025058597f47ca0 Mon Sep 17 00:00:00 2001 From: Bill Nottingham Date: Mon, 11 May 2020 13:11:19 -0400 Subject: [PATCH 2/6] Don't uninstall pycurl, but do build it from source. oVirt requires something newer, but we can't use the wheel. --- Makefile | 2 +- requirements/requirements_ansible_uninstall.txt | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/Makefile b/Makefile index dba1e0844b..d14d44cfc3 100644 --- a/Makefile +++ b/Makefile @@ -24,7 +24,7 @@ CELERY_SCHEDULE_FILE ?= /var/lib/awx/beat.db DEV_DOCKER_TAG_BASE ?= gcr.io/ansible-tower-engineering # Python packages to install only from source (not from binary wheels) # Comma separated list -SRC_ONLY_PKGS ?= cffi,pycparser,psycopg2,twilio +SRC_ONLY_PKGS ?= cffi,pycparser,psycopg2,twilio,pycurl # These should be upgraded in the AWX and Ansible venv before attempting # to install the actual requirements VENV_BOOTSTRAP ?= pip==19.3.1 setuptools==41.6.0 diff --git a/requirements/requirements_ansible_uninstall.txt b/requirements/requirements_ansible_uninstall.txt index 53a2b2ee08..56cbaa5f19 100644 --- a/requirements/requirements_ansible_uninstall.txt +++ b/requirements/requirements_ansible_uninstall.txt @@ -1,2 +1 @@ -pycurl # requires system package version rsa # stop adding new crypto libs From 725437571d8447fc54bb7341f5b4727766a418bd Mon Sep 17 00:00:00 2001 From: Bill Nottingham Date: Mon, 11 May 2020 14:21:09 -0400 Subject: [PATCH 3/6] Set a default pycurl SSL backend. --- Makefile | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index d14d44cfc3..3946b76f02 100644 --- a/Makefile +++ b/Makefile @@ -12,6 +12,7 @@ MANAGEMENT_COMMAND ?= awx-manage IMAGE_REPOSITORY_AUTH ?= IMAGE_REPOSITORY_BASE ?= https://gcr.io VERSION := $(shell cat VERSION) +PYCURL_SSL_LIBRARY ?= openssl # NOTE: This defaults the container image version to the branch that's active COMPOSE_TAG ?= $(GIT_BRANCH) @@ -173,9 +174,9 @@ virtualenv_awx: # --ignore-install flag is not used because *.txt files should specify exact versions requirements_ansible: virtualenv_ansible if [[ "$(PIP_OPTIONS)" == *"--no-index"* ]]; then \ - cat requirements/requirements_ansible.txt requirements/requirements_ansible_local.txt | $(VENV_BASE)/ansible/bin/pip install $(PIP_OPTIONS) -r /dev/stdin ; \ + cat requirements/requirements_ansible.txt requirements/requirements_ansible_local.txt | PYCURL_SSL_LIBRARY=$(PYCURL_SSL_LIBRARY) $(VENV_BASE)/ansible/bin/pip install $(PIP_OPTIONS) -r /dev/stdin ; \ else \ - cat requirements/requirements_ansible.txt requirements/requirements_ansible_git.txt | $(VENV_BASE)/ansible/bin/pip install $(PIP_OPTIONS) --no-binary $(SRC_ONLY_PKGS) -r /dev/stdin ; \ + cat requirements/requirements_ansible.txt requirements/requirements_ansible_git.txt | PYCURL_SSL_LIBRARY=$(PYCURL_SSL_LIBRARY) $(VENV_BASE)/ansible/bin/pip install $(PIP_OPTIONS) --no-binary $(SRC_ONLY_PKGS) -r /dev/stdin ; \ fi $(VENV_BASE)/ansible/bin/pip uninstall --yes -r requirements/requirements_ansible_uninstall.txt # Same effect as using --system-site-packages flag on venv creation @@ -183,9 +184,9 @@ requirements_ansible: virtualenv_ansible requirements_ansible_py3: virtualenv_ansible_py3 if [[ "$(PIP_OPTIONS)" == *"--no-index"* ]]; then \ - cat requirements/requirements_ansible.txt requirements/requirements_ansible_local.txt | $(VENV_BASE)/ansible/bin/pip3 install $(PIP_OPTIONS) -r /dev/stdin ; \ + cat requirements/requirements_ansible.txt requirements/requirements_ansible_local.txt | PYCURL_SSL_LIBRARY=$(PYCURL_SSL_LIBRARY) $(VENV_BASE)/ansible/bin/pip3 install $(PIP_OPTIONS) -r /dev/stdin ; \ else \ - cat requirements/requirements_ansible.txt requirements/requirements_ansible_git.txt | $(VENV_BASE)/ansible/bin/pip3 install $(PIP_OPTIONS) --no-binary $(SRC_ONLY_PKGS) -r /dev/stdin ; \ + cat requirements/requirements_ansible.txt requirements/requirements_ansible_git.txt | PYCURL_SSL_LIBRARY=$(PYCURL_SSL_LIBRARY) $(VENV_BASE)/ansible/bin/pip3 install $(PIP_OPTIONS) --no-binary $(SRC_ONLY_PKGS) -r /dev/stdin ; \ fi $(VENV_BASE)/ansible/bin/pip3 uninstall --yes -r requirements/requirements_ansible_uninstall.txt # Same effect as using --system-site-packages flag on venv creation From 2e2fe40d2a6a87eb6bbb31f33a92af174406ba9e Mon Sep 17 00:00:00 2001 From: Alan Rominger Date: Mon, 11 May 2020 22:45:19 -0400 Subject: [PATCH 4/6] Add options to ovirt inventory file (#4307) fixes schema differences from script add back in default groups from script change hostnames to reflect script add in some hostvars Generally allow giving plugin options from source variables allows testing with insecure connection with ovirt_insecure this is a behavior change from the script --- awx/main/models/inventory.py | 19 +++++++++++++++++++ .../inventory/plugins/rhv/files/ovirt.yml | 19 +++++++++++++++++++ .../test_inventory_source_injectors.py | 7 ++++++- 3 files changed, 44 insertions(+), 1 deletion(-) diff --git a/awx/main/models/inventory.py b/awx/main/models/inventory.py index 06fbd071e7..6e5ffce508 100644 --- a/awx/main/models/inventory.py +++ b/awx/main/models/inventory.py @@ -2500,6 +2500,25 @@ class rhv(PluginFileInjector): def script_name(self): return 'ovirt4.py' # exception + def inventory_as_dict(self, inventory_update, private_data_dir): + ret = super(rhv, self).inventory_as_dict(inventory_update, private_data_dir) + ret['ovirt_insecure'] = False # Default changed from script + # TODO: process strict option upstream + ret['compose'] = { + 'ansible_host': '(devices.values() | list)[0][0] if devices else None' + } + ret['keyed_groups'] = [] + for key in ('cluster', 'status'): + ret['keyed_groups'].append({'prefix': key, 'separator': '_', 'key': key}) + ret['keyed_groups'].append({'prefix': 'tag', 'separator': '_', 'key': 'tags'}) + ret['ovirt_hostname_preference'] = ['name', 'fqdn'] + source_vars = inventory_update.source_vars_dict + for key, value in source_vars.items(): + if key == 'plugin': + continue + ret[key] = value + return ret + class satellite6(PluginFileInjector): plugin_name = 'foreman' diff --git a/awx/main/tests/data/inventory/plugins/rhv/files/ovirt.yml b/awx/main/tests/data/inventory/plugins/rhv/files/ovirt.yml index 0c7dec7b16..67a94ae6de 100644 --- a/awx/main/tests/data/inventory/plugins/rhv/files/ovirt.yml +++ b/awx/main/tests/data/inventory/plugins/rhv/files/ovirt.yml @@ -1 +1,20 @@ +base_source_var: value_of_var +compose: + ansible_host: (devices.values() | list)[0][0] if devices else None +groups: + dev: '"dev" in tags' +keyed_groups: +- key: cluster + prefix: cluster + separator: _ +- key: status + prefix: status + separator: _ +- key: tags + prefix: tag + separator: _ +ovirt_hostname_preference: +- name +- fqdn +ovirt_insecure: false plugin: ovirt.ovirt.ovirt diff --git a/awx/main/tests/functional/test_inventory_source_injectors.py b/awx/main/tests/functional/test_inventory_source_injectors.py index 29b798cfd7..4e549e1be6 100644 --- a/awx/main/tests/functional/test_inventory_source_injectors.py +++ b/awx/main/tests/functional/test_inventory_source_injectors.py @@ -50,7 +50,6 @@ INI_TEST_VARS = { 'expand_hostvars': True, 'fail_on_errors': True }, - 'rhv': {}, # there are none 'tower': {}, # there are none 'vmware': { 'alias_pattern': "{{ config.foo }}", @@ -78,6 +77,12 @@ INI_TEST_VARS = { 'nest_tags': 'yes', 'suffix': '.ppt', 'prefer_ipv4': 'yes' + }, + 'rhv': { # options specific to the plugin + 'ovirt_insecure': False, + 'groups': { + 'dev': '"dev" in tags' + } } } From 732f2fb8283a112b7467d14f4720973b3c678101 Mon Sep 17 00:00:00 2001 From: ansible-translation-bot Date: Tue, 12 May 2020 13:15:21 +0000 Subject: [PATCH 5/6] UI translation strings for release_3.7.0 branch --- awx/locale/ja/LC_MESSAGES/django.po | 14 +++++++------- awx/ui/po/ja.po | 25 ++++++++++++------------- 2 files changed, 19 insertions(+), 20 deletions(-) diff --git a/awx/locale/ja/LC_MESSAGES/django.po b/awx/locale/ja/LC_MESSAGES/django.po index 6e08b48e53..db72a7ec8f 100644 --- a/awx/locale/ja/LC_MESSAGES/django.po +++ b/awx/locale/ja/LC_MESSAGES/django.po @@ -777,13 +777,13 @@ msgstr "このグループ内でインスタンスをコンテナー化するか #: awx/api/serializers.py:4683 msgid "Policy Instance Percentage" -msgstr "ポリシーインスタンスのパーセンテージ" +msgstr "ポリシーインスタンスの割合" #: awx/api/serializers.py:4684 msgid "" "Minimum percentage of all instances that will be automatically assigned to " "this group when new instances come online." -msgstr "新規インスタンスがオンライン状態になるとこのグループに自動的に割り当てられるすべてのインスタンスの最小パーセンテージです。" +msgstr "新規インスタンスがオンラインになると、このグループに自動的に最小限割り当てられるインスタンスの割合を選択します。" #: awx/api/serializers.py:4689 msgid "Policy Instance Minimum" @@ -793,7 +793,7 @@ msgstr "ポリシーインスタンスの最小値" msgid "" "Static minimum number of Instances that will be automatically assign to this " "group when new instances come online." -msgstr "新規インスタンスがオンライン状態になるとこのグループに自動的に割り当てられるインスタンスの静的な最小数です。" +msgstr "新規インスタンスがオンラインになると、このグループに自動的に最小限割り当てられるインスタンス数を入力します。" #: awx/api/serializers.py:4695 msgid "Policy Instance List" @@ -1886,7 +1886,7 @@ msgstr "組織管理者が、それぞれの組織に関連付けられていな #: awx/main/conf.py:50 msgid "Organization Admins Can Manage Users and Teams" -msgstr "組織管理者はユーザーおよびチームを管理できます" +msgstr "組織管理者はユーザーおよびチームを管理できる" #: awx/main/conf.py:51 msgid "" @@ -2243,7 +2243,7 @@ msgid "" "downloaded from the primary if the specific role or collection is available " "there. If the content is not avilable in the primary, or if this field is " "left blank, it will default to galaxy.ansible.com." -msgstr "これにより、Galaxy サービスを実行する組織では、プライマリーの Galaxy Server としてホストを指定するオプションが追加されます。特定のロールやコレクションが利用できる場合には、このプライマリーのサーバーから要件がダウンロードされます。または、フィールドが空白の場合は、galaxy.ansible.com のデフォルト設定を使用します。" +msgstr "Galaxy サービスを実行する組織では、プライマリーの Galaxy Server としてホストを指定するオプションが追加されます。特定のロールやコレクションが利用できる場合には、このプライマリーのサーバーから要件がダウンロードされます。または、フィールドが空白の場合は、galaxy.ansible.com のデフォルト設定を使用します。" #: awx/main/conf.py:459 msgid "Primary Galaxy Server Username" @@ -3448,7 +3448,7 @@ msgstr "認証する OpenShift または Kubernetes API エンドポイント。 #: awx/main/models/credential/__init__.py:1139 msgid "API authentication bearer token" -msgstr "API 認証ベアラートークン。" +msgstr "API 認証ベアラートークン" #: awx/main/models/credential/__init__.py:1149 msgid "Certificate Authority data" @@ -3605,7 +3605,7 @@ msgstr "このグループをリモートで制御するためのインスタン #: awx/main/models/ha.py:200 msgid "Percentage of Instances to automatically assign to this group" -msgstr "このグループに自動的に割り当てるインスタンスのパーセンテージ" +msgstr "このグループに自動的に割り当てるインスタンスの割合" #: awx/main/models/ha.py:204 msgid "" diff --git a/awx/ui/po/ja.po b/awx/ui/po/ja.po index c9a1d1e45e..7e44693fb0 100644 --- a/awx/ui/po/ja.po +++ b/awx/ui/po/ja.po @@ -67,7 +67,7 @@ msgid "" " " msgstr "\n" "
\n" -" ログアウトしています。サインインしてください。\n" +" ログアウトされました。サインインしてください。\n" "
" #: client/lib/components/code-mirror/code-mirror.strings.js:19 @@ -478,7 +478,7 @@ msgstr "すべてのジョブ" #: client/src/projects/projects.form.js:208 #: client/src/projects/projects.form.js:211 msgid "Allow branch override" -msgstr "ブランチの上書きを許可する" +msgstr "ブランチの上書き許可" #: client/src/projects/projects.form.js:210 msgid "Allow changing the SCM branch or revision in a job template that uses this project." @@ -916,7 +916,7 @@ msgstr "キャッシュタイムアウト" #: client/src/projects/projects.form.js:220 msgid "Cache Timeout%s (seconds)%s" -msgstr "キャッシュタイムアウト%s (seconds)%s" +msgstr "キャッシュタイムアウト%s (秒)%s" #: client/src/users/list/users-list.controller.js:85 msgid "Call to %s failed. DELETE returned status:" @@ -1191,11 +1191,11 @@ msgstr "CloudForms URL" #: client/features/output/output.strings.js:19 #: client/src/workflow-results/workflow-results.controller.js:201 msgid "Collapse Output" -msgstr "出力の縮小" +msgstr "出力の折りたたみ" #: client/lib/components/components.strings.js:120 msgid "Compact" -msgstr "縮小" +msgstr "簡略表示" #: client/src/inventories-hosts/hosts/host.form.js:125 #: client/src/inventories-hosts/inventories/related/hosts/related-host.form.js:135 @@ -1307,8 +1307,7 @@ msgstr "{{resourceType}} のコピー" msgid "" "Copyright © 2019 Red Hat, Inc.
\n" " Visit Ansible.com for more information.
" -msgstr "Copyright © 2019 Red Hat, Inc.
\n" -" 詳細は、 Ansible.com をご覧ください。
" +msgstr "Copyright © 2019 Red Hat, Inc.
詳細は、 Ansible.com をご覧ください。
" #: client/lib/components/components.strings.js:90 msgid "Copyright © 2019 Red Hat, Inc." @@ -2128,7 +2127,7 @@ msgstr "このワークフローのジョブテンプレートの Webhook を有 #: client/src/templates/job_templates/job-template.form.js:335 msgid "Enables creation of a provisioning callback URL. Using the URL a host can contact {{BRAND_NAME}} and request a configuration update using this job template." -msgstr "プロビジョニングコールバック URL の作成を有効にします。この URL を使用してホストは {{BRAND_NAME}} に接続でき、このジョブテンプレートを使用して設定の更新を要求できます。" +msgstr "プロビジョニングコールバック URL の作成を有効にします。ホストは、この URL を使用して {{BRAND_NAME}} に接続でき、このジョブテンプレートを使用して設定の更新を要求できます。" #: client/src/credentials/factories/credential-form-save.factory.js:73 msgid "Encrypted credentials are not supported." @@ -2368,7 +2367,7 @@ msgstr "出力の展開" #: client/lib/components/components.strings.js:121 msgid "Expanded" -msgstr "展開" +msgstr "詳細表示" #: client/src/license/license.partial.html:246 msgid "Expires" @@ -5372,7 +5371,7 @@ msgstr "Insights の更新" #: client/src/configuration/settings.service.js:44 msgid "Refresh Token Expiration" -msgstr "トークンの有効期限を更新します。" +msgstr "トークンの有効期限を更新する" #: client/src/activity-stream/streams.list.js:51 #: client/src/bread-crumb/bread-crumb.partial.html:6 @@ -7281,7 +7280,7 @@ msgstr "TLS の使用" #: client/src/notifications/notificationTemplates.form.js:586 msgid "Use custom messages to change the content of notifications sent when a job starts, succeeds, or fails. Use curly braces to access information about the job: {{ job_friendly_name }}, {{ url }}, or attributes of the job such as {{ job.status }}. You may apply a number of possible variables in the message. Refer to the Ansible Tower documentation for more details." -msgstr "カスタムメッセージを使用して、ジョブの開始時、成功時、または湿板時に送信する通知内容を変更します。中括弧を使用して、ジョブの情報 ({{ job_friendly_name }}, {{ url }}) またはジョブの属性 ({{ job.status }}) にアクセスします。メッセージには、使用可能な変数を複数適用できます。詳細は、Ansible Tower ドキュメント を参照してください。" +msgstr "カスタムメッセージを使用して、ジョブの開始時、成功時、または失敗時に送信する通知内容を変更します。中括弧を使用して、ジョブの情報 ({{ job_friendly_name }}, {{ url }}) またはジョブの属性 ({{ job.status }}) にアクセスします。メッセージには、使用可能な変数を複数適用できます。詳細は、Ansible Tower ドキュメント を参照してください。" #: client/src/instance-groups/instance-groups.strings.js:24 #: client/src/instance-groups/instance-groups.strings.js:50 @@ -7509,7 +7508,7 @@ msgstr "YAML サンプルを %s に表示" #: client/src/configuration/settings.partial.html:15 msgid "View and edit your license information" -msgstr "ライセンス情報を表示し、編集します。" +msgstr "ライセンス情報の表示、編集" #: client/src/credentials/credentials.list.js:84 msgid "View credential" @@ -7876,7 +7875,7 @@ msgstr "ホストフィルターの編集に必要なパーミッションがあ #: client/src/login/loginModal/loginModal.partial.html:34 msgid "You have been logged out. Please sign in." -msgstr "ログアウトしています。サインインしてください。" +msgstr "ログアウトされました。サインインしてください。" #: client/src/configuration/forms/auth-form/configuration-auth.controller.js:51 #: client/src/configuration/forms/settings-form.controller.js:443 From c3ff7ab2477e75390b9f224f6929f2535fcf7266 Mon Sep 17 00:00:00 2001 From: Shane McDonald Date: Tue, 12 May 2020 16:35:09 -0400 Subject: [PATCH 6/6] Do not start services when generating swagger docs --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 3946b76f02..1e31f75e0b 100644 --- a/Makefile +++ b/Makefile @@ -641,7 +641,7 @@ docker-compose-runtest: awx/projects cd tools && CURRENT_UID=$(shell id -u) TAG=$(COMPOSE_TAG) DEV_DOCKER_TAG_BASE=$(DEV_DOCKER_TAG_BASE) docker-compose run --rm --service-ports awx /start_tests.sh docker-compose-build-swagger: awx/projects - cd tools && CURRENT_UID=$(shell id -u) TAG=$(COMPOSE_TAG) DEV_DOCKER_TAG_BASE=$(DEV_DOCKER_TAG_BASE) docker-compose run --rm --service-ports awx /start_tests.sh swagger + cd tools && CURRENT_UID=$(shell id -u) TAG=$(COMPOSE_TAG) DEV_DOCKER_TAG_BASE=$(DEV_DOCKER_TAG_BASE) docker-compose run --rm --service-ports --no-deps awx /start_tests.sh swagger detect-schema-change: genschema curl https://s3.amazonaws.com/awx-public-ci-files/schema.json -o reference-schema.json