From c1f6fec53216dcf1837ae89f00c6d6746e513790 Mon Sep 17 00:00:00 2001 From: Jeff Bradberry Date: Wed, 22 Jan 2020 16:09:53 -0500 Subject: [PATCH 01/73] Add skeleton of a custom 'export' resource This only takes a flag for users at the moment, and does nothing in particular with it. --- awxkit/awxkit/cli/resource.py | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/awxkit/awxkit/cli/resource.py b/awxkit/awxkit/cli/resource.py index c70aa5050d..11a5c5e6b8 100644 --- a/awxkit/awxkit/cli/resource.py +++ b/awxkit/awxkit/cli/resource.py @@ -123,6 +123,41 @@ class Config(CustomCommand): } +class Export(CustomCommand): + name = 'export' + help_text = 'export resources from Tower as yaml' + + def extend_parser(self, parser): + resources = parser.add_argument_group('resources') + resources.add_argument('--users', nargs='?', const='') + + def handle(self, client, parser): + self.extend_parser(parser) + + if client.help: + parser.print_help() + raise SystemExit() + + parsed = parser.parse_known_args()[0] + + data = {} + for resource in ('users',): + value = getattr(parsed, resource, None) + if value is None: + print("Pulling no users.") + continue + if value: + print("Pulling users: {}".format(value)) + pass + else: + print("Pulling all users.") + pass + + data[resource] = {} + + return data + + def parse_resource(client, skip_deprecated=False): subparsers = client.parser.add_subparsers( dest='resource', From 4312395a3afdab0604e071e5052db1a93a3f6276 Mon Sep 17 00:00:00 2001 From: Jeff Bradberry Date: Mon, 27 Jan 2020 16:16:34 -0500 Subject: [PATCH 02/73] Normalize the requested resource into a pk using the pk_or_name helper function. Also, authenticate to the API. --- awxkit/awxkit/cli/resource.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/awxkit/awxkit/cli/resource.py b/awxkit/awxkit/cli/resource.py index 11a5c5e6b8..5baecc7ec1 100644 --- a/awxkit/awxkit/cli/resource.py +++ b/awxkit/awxkit/cli/resource.py @@ -131,6 +131,14 @@ class Export(CustomCommand): resources = parser.add_argument_group('resources') resources.add_argument('--users', nargs='?', const='') + def get_resources(self, client, resource, value): + if value: + from .options import pk_or_name + + print("Pulling {}: {}".format(resource, pk_or_name(client.v2, resource, value))) + else: + print("Pulling all {}.".format(resource)) + def handle(self, client, parser): self.extend_parser(parser) @@ -138,20 +146,16 @@ class Export(CustomCommand): parser.print_help() raise SystemExit() + client.authenticate() parsed = parser.parse_known_args()[0] data = {} for resource in ('users',): value = getattr(parsed, resource, None) if value is None: - print("Pulling no users.") + print("Pulling no {}.".format(resource)) continue - if value: - print("Pulling users: {}".format(value)) - pass - else: - print("Pulling all users.") - pass + self.get_resources(client, resource, value) data[resource] = {} From f21d6b1fc4a947807b2ff5d5001e237be415a778 Mon Sep 17 00:00:00 2001 From: Jeff Bradberry Date: Tue, 28 Jan 2020 11:07:16 -0500 Subject: [PATCH 03/73] Actually query the API for the user or users requested --- awxkit/awxkit/cli/resource.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/awxkit/awxkit/cli/resource.py b/awxkit/awxkit/cli/resource.py index 5baecc7ec1..5629742511 100644 --- a/awxkit/awxkit/cli/resource.py +++ b/awxkit/awxkit/cli/resource.py @@ -132,12 +132,14 @@ class Export(CustomCommand): resources.add_argument('--users', nargs='?', const='') def get_resources(self, client, resource, value): + api_resource = getattr(client.v2, resource) if value: from .options import pk_or_name - print("Pulling {}: {}".format(resource, pk_or_name(client.v2, resource, value))) + pk = pk_or_name(client.v2, resource, value) + return api_resource.get(id=pk).json['results'] else: - print("Pulling all {}.".format(resource)) + return api_resource.get(all_pages=True).json['results'] def handle(self, client, parser): self.extend_parser(parser) @@ -153,11 +155,10 @@ class Export(CustomCommand): for resource in ('users',): value = getattr(parsed, resource, None) if value is None: - print("Pulling no {}.".format(resource)) continue - self.get_resources(client, resource, value) + resources = self.get_resources(client, resource, value) or [] - data[resource] = {} + data[resource] = resources return data From c1a07ff00b08ad35674e979b3f6f675e1a8f34bf Mon Sep 17 00:00:00 2001 From: Jeff Bradberry Date: Wed, 29 Jan 2020 13:21:13 -0500 Subject: [PATCH 04/73] Limit export output to only those fields needed to create the resource --- awxkit/awxkit/cli/resource.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/awxkit/awxkit/cli/resource.py b/awxkit/awxkit/cli/resource.py index 5629742511..c899126f96 100644 --- a/awxkit/awxkit/cli/resource.py +++ b/awxkit/awxkit/cli/resource.py @@ -133,13 +133,16 @@ class Export(CustomCommand): def get_resources(self, client, resource, value): api_resource = getattr(client.v2, resource) + post_fields = api_resource.options().json['actions']['POST'] if value: from .options import pk_or_name pk = pk_or_name(client.v2, resource, value) - return api_resource.get(id=pk).json['results'] + results = api_resource.get(id=pk).json['results'] else: - return api_resource.get(all_pages=True).json['results'] + results = api_resource.get(all_pages=True).json['results'] + + return [{key: r[key] for key in post_fields if key in r} for r in results] def handle(self, client, parser): self.extend_parser(parser) @@ -156,7 +159,7 @@ class Export(CustomCommand): value = getattr(parsed, resource, None) if value is None: continue - resources = self.get_resources(client, resource, value) or [] + resources = self.get_resources(client, resource, value) data[resource] = resources From 9f7fecf8da117bf91e819a6a330711ad3ffcf995 Mon Sep 17 00:00:00 2001 From: Jeff Bradberry Date: Wed, 29 Jan 2020 14:31:38 -0500 Subject: [PATCH 05/73] Add basic import command --- awxkit/awxkit/cli/resource.py | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/awxkit/awxkit/cli/resource.py b/awxkit/awxkit/cli/resource.py index c899126f96..47b8c83a39 100644 --- a/awxkit/awxkit/cli/resource.py +++ b/awxkit/awxkit/cli/resource.py @@ -1,4 +1,6 @@ +import json import os +import sys from awxkit import api, config from awxkit.utils import to_str @@ -123,9 +125,34 @@ class Config(CustomCommand): } +class Import(CustomCommand): + name = 'import' + help_text = 'import resources into Tower' + + def create_resource(self, client, resource, asset): + api_resource = getattr(client.v2, resource) + if resource == 'users' and 'password' not in asset: + asset['password'] = 'password' + api_resource.post(asset) + + def handle(self, client, parser): + if client.help: + parser.print_help() + raise SystemExit() + + data = json.load(sys.stdin) + client.authenticate() + + for resource, assets in data.items(): + for asset in assets: + self.create_resource(client, resource, asset) + + return {} + + class Export(CustomCommand): name = 'export' - help_text = 'export resources from Tower as yaml' + help_text = 'export resources from Tower' def extend_parser(self, parser): resources = parser.add_argument_group('resources') From 2127f3c96dda39fa591d09231c9f067df080f5f6 Mon Sep 17 00:00:00 2001 From: Jeff Bradberry Date: Wed, 29 Jan 2020 14:41:39 -0500 Subject: [PATCH 06/73] Add organizations as a resource type to export --- awxkit/awxkit/cli/resource.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/awxkit/awxkit/cli/resource.py b/awxkit/awxkit/cli/resource.py index 47b8c83a39..107dccf86c 100644 --- a/awxkit/awxkit/cli/resource.py +++ b/awxkit/awxkit/cli/resource.py @@ -157,6 +157,7 @@ class Export(CustomCommand): def extend_parser(self, parser): resources = parser.add_argument_group('resources') resources.add_argument('--users', nargs='?', const='') + resources.add_argument('--organizations', nargs='?', const='') def get_resources(self, client, resource, value): api_resource = getattr(client.v2, resource) @@ -182,7 +183,7 @@ class Export(CustomCommand): parsed = parser.parse_known_args()[0] data = {} - for resource in ('users',): + for resource in ('users', 'organizations'): value = getattr(parsed, resource, None) if value is None: continue From 9280198b0f00d2d43732e7a2205418bac11bed59 Mon Sep 17 00:00:00 2001 From: Jeff Bradberry Date: Thu, 30 Jan 2020 16:50:16 -0500 Subject: [PATCH 07/73] Use a list of exportable resources to build up the parser Also, handle an implicit export of everything. --- awxkit/awxkit/cli/resource.py | 34 ++++++++++++++++++++++++++++------ 1 file changed, 28 insertions(+), 6 deletions(-) diff --git a/awxkit/awxkit/cli/resource.py b/awxkit/awxkit/cli/resource.py index 107dccf86c..1a4af456a7 100644 --- a/awxkit/awxkit/cli/resource.py +++ b/awxkit/awxkit/cli/resource.py @@ -44,6 +44,20 @@ DEPRECATED_RESOURCES_REVERSE = dict( (v, k) for k, v in DEPRECATED_RESOURCES.items() ) +EXPORTABLE_RESOURCES = [ + 'users', + 'organizations', + 'teams', + # 'credential_types', + # 'credentials', + # 'notification_templates', + # 'inventory_scripts', + # 'projects', + # 'inventory', + # 'job_templates', + # 'workflow_job_templates', +] + class CustomCommand(metaclass=CustomRegistryMeta): """Base class for implementing custom commands. @@ -156,12 +170,17 @@ class Export(CustomCommand): def extend_parser(self, parser): resources = parser.add_argument_group('resources') - resources.add_argument('--users', nargs='?', const='') - resources.add_argument('--organizations', nargs='?', const='') + + for resource in EXPORTABLE_RESOURCES: + # This parsing pattern will result in 3 different possible outcomes: + # 1) the resource flag is not used at all, which will result in the attr being None + # 2) the resource flag is used with no argument, which will result in the attr being '' + # 3) the resource flag is used with an argument, and the attr will be that argument's value + resources.add_argument('--{}'.format(resource), nargs='?', const='') def get_resources(self, client, resource, value): api_resource = getattr(client.v2, resource) - post_fields = api_resource.options().json['actions']['POST'] + post_options = api_resource.options().json['actions']['POST'] if value: from .options import pk_or_name @@ -170,7 +189,7 @@ class Export(CustomCommand): else: results = api_resource.get(all_pages=True).json['results'] - return [{key: r[key] for key in post_fields if key in r} for r in results] + return [{key: r[key] for key in post_options if key in r} for r in results] def handle(self, client, parser): self.extend_parser(parser) @@ -182,10 +201,13 @@ class Export(CustomCommand): client.authenticate() parsed = parser.parse_known_args()[0] + # If no resource flags are explicitly used, export everything. + all_resources = all(getattr(parsed, resource, None) is None for resource in EXPORTABLE_RESOURCES) + data = {} - for resource in ('users', 'organizations'): + for resource in EXPORTABLE_RESOURCES: value = getattr(parsed, resource, None) - if value is None: + if value is None and not all_resources: continue resources = self.get_resources(client, resource, value) From 37cbf7691b8d67fb6931088e8a7b5c9b3e195fba Mon Sep 17 00:00:00 2001 From: Jeff Bradberry Date: Mon, 3 Feb 2020 11:06:08 -0500 Subject: [PATCH 08/73] Split out separate methods for the OPTIONS call and massaging each asset dict --- awxkit/awxkit/cli/resource.py | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/awxkit/awxkit/cli/resource.py b/awxkit/awxkit/cli/resource.py index 1a4af456a7..45415e2378 100644 --- a/awxkit/awxkit/cli/resource.py +++ b/awxkit/awxkit/cli/resource.py @@ -178,18 +178,25 @@ class Export(CustomCommand): # 3) the resource flag is used with an argument, and the attr will be that argument's value resources.add_argument('--{}'.format(resource), nargs='?', const='') - def get_resources(self, client, resource, value): - api_resource = getattr(client.v2, resource) - post_options = api_resource.options().json['actions']['POST'] + def get_resource_options(self, endpoint): + return endpoint.options().json['actions']['POST'] + + def get_assets(self, endpoint, value): if value: from .options import pk_or_name pk = pk_or_name(client.v2, resource, value) - results = api_resource.get(id=pk).json['results'] + results = endpoint.get(id=pk).json['results'] else: - results = api_resource.get(all_pages=True).json['results'] + results = endpoint.get(all_pages=True).json['results'] - return [{key: r[key] for key in post_options if key in r} for r in results] + return results + + def enhance_asset(self, endpoint, asset, options): + fields = {key: asset[key] for key in options if key in asset} + fk_fields = {} + related_fields = {} + return dict(**fields, **fk_fields, **related_fields) def handle(self, client, parser): self.extend_parser(parser) @@ -209,9 +216,11 @@ class Export(CustomCommand): value = getattr(parsed, resource, None) if value is None and not all_resources: continue - resources = self.get_resources(client, resource, value) + endpoint = getattr(client.v2, resource) + options = self.get_resource_options(endpoint) + assets = self.get_assets(endpoint, value) - data[resource] = resources + data[resource] = [self.enhance_asset(endpoint, asset, options) for asset in assets] return data From dfe34563aa1b092e81aafb946d22565f4d061652 Mon Sep 17 00:00:00 2001 From: Jeff Bradberry Date: Tue, 11 Feb 2020 14:43:30 -0500 Subject: [PATCH 09/73] Resolve the natural keys of assets we've already collected --- awxkit/awxkit/cli/resource.py | 60 ++++++++++++++++++++++++++++++++--- 1 file changed, 56 insertions(+), 4 deletions(-) diff --git a/awxkit/awxkit/cli/resource.py b/awxkit/awxkit/cli/resource.py index 45415e2378..8d08ffdf3f 100644 --- a/awxkit/awxkit/cli/resource.py +++ b/awxkit/awxkit/cli/resource.py @@ -51,13 +51,28 @@ EXPORTABLE_RESOURCES = [ # 'credential_types', # 'credentials', # 'notification_templates', - # 'inventory_scripts', # 'projects', # 'inventory', # 'job_templates', # 'workflow_job_templates', ] +NATURAL_KEYS = { + 'user': ('username',), + 'organization': ('name',), + 'team': ('organization', 'name'), + 'credential_type': ('name', 'kind'), + 'credential': ('organization', 'name', 'credential_type'), + 'notification_template': ('organization', 'name'), + 'project': ('organization', 'name'), + 'inventory': ('organization', 'name'), + 'job_template': ('name',), + 'workflow_job_template': ('organization', 'name'), + + # related resources + 'role': ('name',), # FIXME: we also need the content_object, itself as a natural key representation +} + class CustomCommand(metaclass=CustomRegistryMeta): """Base class for implementing custom commands. @@ -168,6 +183,10 @@ class Export(CustomCommand): name = 'export' help_text = 'export resources from Tower' + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._natural_keys = {} + def extend_parser(self, parser): resources = parser.add_argument_group('resources') @@ -181,6 +200,16 @@ class Export(CustomCommand): def get_resource_options(self, endpoint): return endpoint.options().json['actions']['POST'] + def register_natural_key(self, asset): + natural_key = {'type': asset['type']} + lookup = NATURAL_KEYS.get(asset['type']) + if callable(lookup): + natural_key.update(lookup(asset)) + else: + natural_key.update((key, asset[key]) for key in lookup or ()) + + self._natural_keys[asset['url']] = natural_key + def get_assets(self, endpoint, value): if value: from .options import pk_or_name @@ -190,12 +219,35 @@ class Export(CustomCommand): else: results = endpoint.get(all_pages=True).json['results'] + for asset in results: + self.register_natural_key(asset) + return results + def get_natural_key(self, asset_url): + if asset_url not in self._natural_keys: + # FIXME: + # get the asset by following the url + # prune down the data using NATURAL_KEYS + # register the natural key dict + return {} + + return self._natural_keys[asset_url] + def enhance_asset(self, endpoint, asset, options): - fields = {key: asset[key] for key in options if key in asset} - fk_fields = {} - related_fields = {} + fields = { + key: asset[key] for key in options + if key in asset and options[key]['type'] != 'id' + } + + fk_fields = { + key: self.get_natural_key(asset['related'][key]) for key in options + if key in asset and options[key]['type'] == 'id' + } + + related = {} + + related_fields = {'related': related} if related else {} return dict(**fields, **fk_fields, **related_fields) def handle(self, client, parser): From 6162ff043931b3da75dd43d3cadcb85977e648e6 Mon Sep 17 00:00:00 2001 From: Jeff Bradberry Date: Tue, 11 Feb 2020 15:16:41 -0500 Subject: [PATCH 10/73] Start to record the role membership but not really, since it still isn't fully capturing the role content_object --- awxkit/awxkit/cli/resource.py | 38 +++++++++++++++++++++++------------ 1 file changed, 25 insertions(+), 13 deletions(-) diff --git a/awxkit/awxkit/cli/resource.py b/awxkit/awxkit/cli/resource.py index 8d08ffdf3f..6af5513557 100644 --- a/awxkit/awxkit/cli/resource.py +++ b/awxkit/awxkit/cli/resource.py @@ -4,7 +4,7 @@ import sys from awxkit import api, config from awxkit.utils import to_str -from awxkit.api.pages import Page +from awxkit.api.pages import Page, TentativePage from awxkit.cli.format import FORMATTERS, format_response, add_authentication_arguments from awxkit.cli.utils import CustomRegistryMeta, cprint @@ -201,15 +201,30 @@ class Export(CustomCommand): return endpoint.options().json['actions']['POST'] def register_natural_key(self, asset): + if asset['url'] in self._natural_keys: + return + natural_key = {'type': asset['type']} lookup = NATURAL_KEYS.get(asset['type']) if callable(lookup): natural_key.update(lookup(asset)) else: - natural_key.update((key, asset[key]) for key in lookup or ()) + natural_key.update((key, asset.get(key)) for key in lookup or ()) self._natural_keys[asset['url']] = natural_key + def get_natural_key(self, url=None, asset=None): + if url is None: + url = asset['url'] + if url not in self._natural_keys: + if asset is None: + # get the asset by following the url + raise Exception("Oops!") + + self.register_natural_key(asset) + + return self._natural_keys[url] + def get_assets(self, endpoint, value): if value: from .options import pk_or_name @@ -224,16 +239,6 @@ class Export(CustomCommand): return results - def get_natural_key(self, asset_url): - if asset_url not in self._natural_keys: - # FIXME: - # get the asset by following the url - # prune down the data using NATURAL_KEYS - # register the natural key dict - return {} - - return self._natural_keys[asset_url] - def enhance_asset(self, endpoint, asset, options): fields = { key: asset[key] for key in options @@ -241,11 +246,18 @@ class Export(CustomCommand): } fk_fields = { - key: self.get_natural_key(asset['related'][key]) for key in options + key: self.get_natural_key(url=asset['related'][key]) for key in options if key in asset and options[key]['type'] == 'id' } related = {} + for k, v in asset['related'].items(): + if k != 'roles': + continue + related_endpoint = TentativePage(v, endpoint.connection) + data = related_endpoint.get(all_pages=True).json + if 'results' in data: + related[k] = [self.get_natural_key(asset=x) for x in data['results']] related_fields = {'related': related} if related else {} return dict(**fields, **fk_fields, **related_fields) From fb066eb52e8bc4e5c446d5fbbee81754be8ddf58 Mon Sep 17 00:00:00 2001 From: Jeff Bradberry Date: Mon, 17 Feb 2020 11:07:52 -0500 Subject: [PATCH 11/73] Fold the other methods doing API calls into get_assets() --- awxkit/awxkit/cli/resource.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/awxkit/awxkit/cli/resource.py b/awxkit/awxkit/cli/resource.py index 6af5513557..a4cc2e4af1 100644 --- a/awxkit/awxkit/cli/resource.py +++ b/awxkit/awxkit/cli/resource.py @@ -225,11 +225,12 @@ class Export(CustomCommand): return self._natural_keys[url] - def get_assets(self, endpoint, value): + def get_assets(self, resource, value): + endpoint = getattr(self.v2, resource) if value: from .options import pk_or_name - pk = pk_or_name(client.v2, resource, value) + pk = pk_or_name(self.v2, resource, value) results = endpoint.get(id=pk).json['results'] else: results = endpoint.get(all_pages=True).json['results'] @@ -237,7 +238,8 @@ class Export(CustomCommand): for asset in results: self.register_natural_key(asset) - return results + options = self.get_resource_options(endpoint) + return [self.enhance_asset(endpoint, asset, options) for asset in results] def enhance_asset(self, endpoint, asset, options): fields = { @@ -275,16 +277,13 @@ class Export(CustomCommand): # If no resource flags are explicitly used, export everything. all_resources = all(getattr(parsed, resource, None) is None for resource in EXPORTABLE_RESOURCES) + self.v2 = client.v2 + data = {} for resource in EXPORTABLE_RESOURCES: value = getattr(parsed, resource, None) - if value is None and not all_resources: - continue - endpoint = getattr(client.v2, resource) - options = self.get_resource_options(endpoint) - assets = self.get_assets(endpoint, value) - - data[resource] = [self.enhance_asset(endpoint, asset, options) for asset in assets] + if all_resources or value is not None: + data[resource] = self.get_assets(resource, value) return data From e4383c505f2cab71fdff3fd8cf9509ade0240d93 Mon Sep 17 00:00:00 2001 From: Jeff Bradberry Date: Mon, 17 Feb 2020 14:37:11 -0500 Subject: [PATCH 12/73] Make more use of the functionality of Page in get_assets and related methods. Also, rename get_resource_options -> get_options and enhance_asset -> serialize_asset. --- awxkit/awxkit/cli/resource.py | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/awxkit/awxkit/cli/resource.py b/awxkit/awxkit/cli/resource.py index a4cc2e4af1..98d6cd8708 100644 --- a/awxkit/awxkit/cli/resource.py +++ b/awxkit/awxkit/cli/resource.py @@ -197,7 +197,7 @@ class Export(CustomCommand): # 3) the resource flag is used with an argument, and the attr will be that argument's value resources.add_argument('--{}'.format(resource), nargs='?', const='') - def get_resource_options(self, endpoint): + def get_options(self, endpoint): return endpoint.options().json['actions']['POST'] def register_natural_key(self, asset): @@ -231,32 +231,31 @@ class Export(CustomCommand): from .options import pk_or_name pk = pk_or_name(self.v2, resource, value) - results = endpoint.get(id=pk).json['results'] + results = endpoint.get(id=pk).results else: - results = endpoint.get(all_pages=True).json['results'] + results = endpoint.get(all_pages=True).results for asset in results: - self.register_natural_key(asset) + self.register_natural_key(asset.json) - options = self.get_resource_options(endpoint) - return [self.enhance_asset(endpoint, asset, options) for asset in results] + options = self.get_options(endpoint) + return [self.serialize_asset(asset, options) for asset in results] - def enhance_asset(self, endpoint, asset, options): + def serialize_asset(self, asset, options): fields = { - key: asset[key] for key in options - if key in asset and options[key]['type'] != 'id' + key: asset.json[key] for key in options + if key in asset.json and options[key]['type'] != 'id' } fk_fields = { - key: self.get_natural_key(url=asset['related'][key]) for key in options - if key in asset and options[key]['type'] == 'id' + key: self.get_natural_key(url=asset.related[key].endpoint) for key in options + if key in asset.json and options[key]['type'] == 'id' } related = {} - for k, v in asset['related'].items(): + for k, related_endpoint in asset.related.items(): if k != 'roles': continue - related_endpoint = TentativePage(v, endpoint.connection) data = related_endpoint.get(all_pages=True).json if 'results' in data: related[k] = [self.get_natural_key(asset=x) for x in data['results']] From 0877e5305c60301a13a843742cc3f0253eb2a642 Mon Sep 17 00:00:00 2001 From: Jeff Bradberry Date: Mon, 17 Feb 2020 15:16:21 -0500 Subject: [PATCH 13/73] Modify get_natural_key to work on Pages --- awxkit/awxkit/cli/resource.py | 34 ++++++++-------------------------- 1 file changed, 8 insertions(+), 26 deletions(-) diff --git a/awxkit/awxkit/cli/resource.py b/awxkit/awxkit/cli/resource.py index 98d6cd8708..f4f3a837a6 100644 --- a/awxkit/awxkit/cli/resource.py +++ b/awxkit/awxkit/cli/resource.py @@ -200,30 +200,15 @@ class Export(CustomCommand): def get_options(self, endpoint): return endpoint.options().json['actions']['POST'] - def register_natural_key(self, asset): - if asset['url'] in self._natural_keys: - return - - natural_key = {'type': asset['type']} - lookup = NATURAL_KEYS.get(asset['type']) + def get_natural_key(self, page): + natural_key = {'type': page.type} + lookup = NATURAL_KEYS.get(page.type) if callable(lookup): - natural_key.update(lookup(asset)) + natural_key.update(lookup(page)) else: - natural_key.update((key, asset.get(key)) for key in lookup or ()) + natural_key.update((key, page[key]) for key in lookup or ()) - self._natural_keys[asset['url']] = natural_key - - def get_natural_key(self, url=None, asset=None): - if url is None: - url = asset['url'] - if url not in self._natural_keys: - if asset is None: - # get the asset by following the url - raise Exception("Oops!") - - self.register_natural_key(asset) - - return self._natural_keys[url] + return natural_key def get_assets(self, resource, value): endpoint = getattr(self.v2, resource) @@ -235,9 +220,6 @@ class Export(CustomCommand): else: results = endpoint.get(all_pages=True).results - for asset in results: - self.register_natural_key(asset.json) - options = self.get_options(endpoint) return [self.serialize_asset(asset, options) for asset in results] @@ -248,7 +230,7 @@ class Export(CustomCommand): } fk_fields = { - key: self.get_natural_key(url=asset.related[key].endpoint) for key in options + key: self.get_natural_key(asset.related[key].get()) for key in options if key in asset.json and options[key]['type'] == 'id' } @@ -258,7 +240,7 @@ class Export(CustomCommand): continue data = related_endpoint.get(all_pages=True).json if 'results' in data: - related[k] = [self.get_natural_key(asset=x) for x in data['results']] + related[k] = [self.get_natural_key(x) for x in data.results] related_fields = {'related': related} if related else {} return dict(**fields, **fk_fields, **related_fields) From 6f28361bf5d63148ec471bc8d2321cd0da6f3e7c Mon Sep 17 00:00:00 2001 From: Jeff Bradberry Date: Mon, 17 Feb 2020 15:37:12 -0500 Subject: [PATCH 14/73] Make serialize_asset use Pages to a greater extent --- awxkit/awxkit/cli/resource.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/awxkit/awxkit/cli/resource.py b/awxkit/awxkit/cli/resource.py index f4f3a837a6..5499bd19c6 100644 --- a/awxkit/awxkit/cli/resource.py +++ b/awxkit/awxkit/cli/resource.py @@ -225,20 +225,20 @@ class Export(CustomCommand): def serialize_asset(self, asset, options): fields = { - key: asset.json[key] for key in options - if key in asset.json and options[key]['type'] != 'id' + key: asset[key] for key in options + if key in asset.json and key not in asset.related } fk_fields = { key: self.get_natural_key(asset.related[key].get()) for key in options - if key in asset.json and options[key]['type'] == 'id' + if key in asset.related } related = {} for k, related_endpoint in asset.related.items(): if k != 'roles': continue - data = related_endpoint.get(all_pages=True).json + data = related_endpoint.get(all_pages=True) if 'results' in data: related[k] = [self.get_natural_key(x) for x in data.results] From 372570ce8e9476b09d4bec866a581ed780072070 Mon Sep 17 00:00:00 2001 From: Jeff Bradberry Date: Mon, 17 Feb 2020 16:04:51 -0500 Subject: [PATCH 15/73] Extract out get_natural_key as a standalone function --- awxkit/awxkit/cli/resource.py | 41 ++++++++++++++++++++--------------- 1 file changed, 24 insertions(+), 17 deletions(-) diff --git a/awxkit/awxkit/cli/resource.py b/awxkit/awxkit/cli/resource.py index 5499bd19c6..80a06c71aa 100644 --- a/awxkit/awxkit/cli/resource.py +++ b/awxkit/awxkit/cli/resource.py @@ -70,10 +70,31 @@ NATURAL_KEYS = { 'workflow_job_template': ('organization', 'name'), # related resources - 'role': ('name',), # FIXME: we also need the content_object, itself as a natural key representation + 'role': ('name', ':content_object'), } +def get_natural_key(page): + natural_key = {'type': page.type} + lookup = NATURAL_KEYS.get(page.type, ()) + + for key in lookup or (): + if key.startswith(':'): + # treat it like a special-case related object + related_objs = [ + related for name, related in page.related.items() + if name not in ('users', 'teams') + ] + if related_objs: + natural_key[key[1:]] = get_natural_key(related_objs[0].get()) + elif key in page.related: + natural_key[key] = get_natural_key(page.related[key].get()) + else: + natural_key[key] = page[key] + + return natural_key + + class CustomCommand(metaclass=CustomRegistryMeta): """Base class for implementing custom commands. @@ -183,10 +204,6 @@ class Export(CustomCommand): name = 'export' help_text = 'export resources from Tower' - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self._natural_keys = {} - def extend_parser(self, parser): resources = parser.add_argument_group('resources') @@ -200,16 +217,6 @@ class Export(CustomCommand): def get_options(self, endpoint): return endpoint.options().json['actions']['POST'] - def get_natural_key(self, page): - natural_key = {'type': page.type} - lookup = NATURAL_KEYS.get(page.type) - if callable(lookup): - natural_key.update(lookup(page)) - else: - natural_key.update((key, page[key]) for key in lookup or ()) - - return natural_key - def get_assets(self, resource, value): endpoint = getattr(self.v2, resource) if value: @@ -230,7 +237,7 @@ class Export(CustomCommand): } fk_fields = { - key: self.get_natural_key(asset.related[key].get()) for key in options + key: get_natural_key(asset.related[key].get()) for key in options if key in asset.related } @@ -240,7 +247,7 @@ class Export(CustomCommand): continue data = related_endpoint.get(all_pages=True) if 'results' in data: - related[k] = [self.get_natural_key(x) for x in data.results] + related[k] = [get_natural_key(x) for x in data.results] related_fields = {'related': related} if related else {} return dict(**fields, **fk_fields, **related_fields) From f53920d3bf238ea721a5a1c2c129fca8fb6c9972 Mon Sep 17 00:00:00 2001 From: Jeff Bradberry Date: Wed, 19 Feb 2020 14:11:27 -0500 Subject: [PATCH 16/73] Begin changing Import to work with the structure created by Export --- awxkit/awxkit/cli/resource.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/awxkit/awxkit/cli/resource.py b/awxkit/awxkit/cli/resource.py index 80a06c71aa..81eb58dac2 100644 --- a/awxkit/awxkit/cli/resource.py +++ b/awxkit/awxkit/cli/resource.py @@ -179,11 +179,12 @@ class Import(CustomCommand): name = 'import' help_text = 'import resources into Tower' - def create_resource(self, client, resource, asset): - api_resource = getattr(client.v2, resource) - if resource == 'users' and 'password' not in asset: - asset['password'] = 'password' - api_resource.post(asset) + def create_assets(self, resource, assets): + endpoint = getattr(self.v2, resource) + for asset in assets: + if resource == 'users' and 'password' not in asset: + asset['password'] = 'password' + endpoint.post({k: v for k, v in asset.items() if k != 'related'}) def handle(self, client, parser): if client.help: @@ -192,10 +193,10 @@ class Import(CustomCommand): data = json.load(sys.stdin) client.authenticate() + self.v2 = client.v2 - for resource, assets in data.items(): - for asset in assets: - self.create_resource(client, resource, asset) + for resource, assets in data.items(): # FIXME: do a topological sort by dependencies + self.create_assets(resource, assets) return {} From 21e36ad19a0c15ba7c7b7e198f12ae1a0d932885 Mon Sep 17 00:00:00 2001 From: Jeff Bradberry Date: Thu, 20 Feb 2020 13:46:03 -0500 Subject: [PATCH 17/73] Add the calculated natural key to the export data for each asset --- awxkit/awxkit/cli/resource.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/awxkit/awxkit/cli/resource.py b/awxkit/awxkit/cli/resource.py index 81eb58dac2..042a8e3572 100644 --- a/awxkit/awxkit/cli/resource.py +++ b/awxkit/awxkit/cli/resource.py @@ -4,7 +4,7 @@ import sys from awxkit import api, config from awxkit.utils import to_str -from awxkit.api.pages import Page, TentativePage +from awxkit.api.pages import Page from awxkit.cli.format import FORMATTERS, format_response, add_authentication_arguments from awxkit.cli.utils import CustomRegistryMeta, cprint @@ -75,8 +75,8 @@ NATURAL_KEYS = { def get_natural_key(page): - natural_key = {'type': page.type} - lookup = NATURAL_KEYS.get(page.type, ()) + natural_key = {'type': page['type']} + lookup = NATURAL_KEYS.get(page['type'], ()) for key in lookup or (): if key.startswith(':'): @@ -236,6 +236,7 @@ class Export(CustomCommand): key: asset[key] for key in options if key in asset.json and key not in asset.related } + fields['natural_key'] = get_natural_key(asset) fk_fields = { key: get_natural_key(asset.related[key].get()) for key in options From 95b22bf05b08dda92ffbfa27bc77f723c73c995d Mon Sep 17 00:00:00 2001 From: Jeff Bradberry Date: Mon, 24 Feb 2020 14:34:55 -0500 Subject: [PATCH 18/73] Begin to deal with existing matching assets by using a registry allowing the lookup of Page objects by (frozen) natural keys. --- awxkit/awxkit/cli/resource.py | 47 +++++++++++++++++++++++++++++++++-- 1 file changed, 45 insertions(+), 2 deletions(-) diff --git a/awxkit/awxkit/cli/resource.py b/awxkit/awxkit/cli/resource.py index 042a8e3572..7f3e95e9ee 100644 --- a/awxkit/awxkit/cli/resource.py +++ b/awxkit/awxkit/cli/resource.py @@ -175,16 +175,51 @@ class Config(CustomCommand): } +def freeze(key): + return frozenset((k, freeze(v) if isinstance(v, dict) else v) for k, v in key.items()) + + class Import(CustomCommand): name = 'import' help_text = 'import resources into Tower' + def __init__(self, *args, **kwargs): + super(Import, self).__init__(*args, **kwargs) + self._natural_key = {} + self._options = {} + def create_assets(self, resource, assets): endpoint = getattr(self.v2, resource) + options = self._options[resource] for asset in assets: + post_data = {} if resource == 'users' and 'password' not in asset: - asset['password'] = 'password' - endpoint.post({k: v for k, v in asset.items() if k != 'related'}) + post_data['password'] = 'password' + for field, value in asset.items(): + if k in ('related', 'natural_key'): + continue + if options[field]['type'] == 'id': + post_data[field] = self._natural_key[freeze(value)]['id'] # FIXME: may not be registered + else: + post_data[field] = value + + natural_key = freeze(asset['natural_key']) + if natural_key in self._natural_key: + page = self._natural_key[natural_key] + page = page.put(post_data) + else: + page = endpoint.post(post_data) + + self._natural_key[freeze(get_natural_key(page))] = page + + def register_existing_assets(self, resource): + endpoint = getattr(self.v2, resource) + options = endpoint.options().json['actions']['POST'] + self._options[resource] = options + + results = endpoint.get(all_pages=True).results + for asset in results: + self._natural_key[freeze(get_natural_key(asset))] = asset def handle(self, client, parser): if client.help: @@ -195,9 +230,17 @@ class Import(CustomCommand): client.authenticate() self.v2 = client.v2 + for resource in data: + self.register_existing_assets(resource) + for resource, assets in data.items(): # FIXME: do a topological sort by dependencies self.create_assets(resource, assets) + # should we delete existing unpatched assets? + + # loop over the sorted assets + # resolve and add in the m2m relateds + return {} From 65e16dc7ae4ecd1abad446ca8cf5ab6f189160de Mon Sep 17 00:00:00 2001 From: Jeff Bradberry Date: Mon, 24 Feb 2020 17:04:30 -0500 Subject: [PATCH 19/73] Get and assign the related objects --- awxkit/awxkit/cli/resource.py | 54 ++++++++++++++++++++++++++++------- 1 file changed, 44 insertions(+), 10 deletions(-) diff --git a/awxkit/awxkit/cli/resource.py b/awxkit/awxkit/cli/resource.py index 7f3e95e9ee..82906fb3f9 100644 --- a/awxkit/awxkit/cli/resource.py +++ b/awxkit/awxkit/cli/resource.py @@ -3,6 +3,7 @@ import os import sys from awxkit import api, config +import awxkit.exceptions as exc from awxkit.utils import to_str from awxkit.api.pages import Page from awxkit.cli.format import FORMATTERS, format_response, add_authentication_arguments @@ -188,6 +189,13 @@ class Import(CustomCommand): self._natural_key = {} self._options = {} + def get_by_natural_key(self, key, fetch=True): + frozen_key = freeze(key) + if frozen_key not in self._natural_key and fetch: + pass + + return self._natural_key.get(frozen_key) + def create_assets(self, resource, assets): endpoint = getattr(self.v2, resource) options = self._options[resource] @@ -196,19 +204,18 @@ class Import(CustomCommand): if resource == 'users' and 'password' not in asset: post_data['password'] = 'password' for field, value in asset.items(): - if k in ('related', 'natural_key'): + if field in ('related', 'natural_key'): continue if options[field]['type'] == 'id': - post_data[field] = self._natural_key[freeze(value)]['id'] # FIXME: may not be registered + post_data[field] = self.get_by_natural_key(value)['id'] else: post_data[field] = value - natural_key = freeze(asset['natural_key']) - if natural_key in self._natural_key: - page = self._natural_key[natural_key] - page = page.put(post_data) - else: + page = self.get_by_natural_key(asset['natural_key'], fetch=False) + if page is None: page = endpoint.post(post_data) + else: + page = page.put(post_data) self._natural_key[freeze(get_natural_key(page))] = page @@ -221,6 +228,33 @@ class Import(CustomCommand): for asset in results: self._natural_key[freeze(get_natural_key(asset))] = asset + def assign_roles(self, page, roles): + role_endpoint = page.json['related']['roles'] + for role in roles: + if 'content_object' not in role: + continue # admin role + obj_page = self.get_by_natural_key(role['content_object']) + if obj_page is not None: + role_page = obj_page.get_object_role(role['name'], by_name=True) + try: + role_endpoint.post({'id': role_page['id']}) + except exc.NoContent: # desired exception on successful (dis)association + pass + else: + pass # admin role + + def assign_related(self, page, name, related_set): + pass + + def assign_related_assets(self, resource, assets): + for asset in assets: + page = self.get_by_natural_key(asset['natural_key']) + for name, S in asset.get('related', {}).items(): + if name == 'roles': + self.assign_roles(page, S) + else: + self.assign_related(page, name, S) + def handle(self, client, parser): if client.help: parser.print_help() @@ -236,10 +270,10 @@ class Import(CustomCommand): for resource, assets in data.items(): # FIXME: do a topological sort by dependencies self.create_assets(resource, assets) - # should we delete existing unpatched assets? + # FIXME: should we delete existing unpatched assets? - # loop over the sorted assets - # resolve and add in the m2m relateds + for resource, assets in data.items(): + self.assign_related_assets(resource, assets) return {} From d191edcaf194e10ef79b5888867cb7e7df34180f Mon Sep 17 00:00:00 2001 From: Jeff Bradberry Date: Fri, 28 Feb 2020 14:21:15 -0500 Subject: [PATCH 20/73] Fix a Python 2 syntax error it doesn't like multiple **kwargs, apparently. --- awxkit/awxkit/cli/resource.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/awxkit/awxkit/cli/resource.py b/awxkit/awxkit/cli/resource.py index 82906fb3f9..821a74f27d 100644 --- a/awxkit/awxkit/cli/resource.py +++ b/awxkit/awxkit/cli/resource.py @@ -329,7 +329,10 @@ class Export(CustomCommand): related[k] = [get_natural_key(x) for x in data.results] related_fields = {'related': related} if related else {} - return dict(**fields, **fk_fields, **related_fields) + + fields.update(fk_fields) + fields.update(related_fields) + return fields def handle(self, client, parser): self.extend_parser(parser) From 2c00d42ced76bed24a5cfc42a5ed6963ede34ae5 Mon Sep 17 00:00:00 2001 From: Jeff Bradberry Date: Mon, 9 Mar 2020 16:00:28 -0400 Subject: [PATCH 21/73] Only use a default password if an import user doesn't already exist --- awxkit/awxkit/cli/resource.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/awxkit/awxkit/cli/resource.py b/awxkit/awxkit/cli/resource.py index 821a74f27d..ad700cc23c 100644 --- a/awxkit/awxkit/cli/resource.py +++ b/awxkit/awxkit/cli/resource.py @@ -201,8 +201,6 @@ class Import(CustomCommand): options = self._options[resource] for asset in assets: post_data = {} - if resource == 'users' and 'password' not in asset: - post_data['password'] = 'password' for field, value in asset.items(): if field in ('related', 'natural_key'): continue @@ -213,6 +211,9 @@ class Import(CustomCommand): page = self.get_by_natural_key(asset['natural_key'], fetch=False) if page is None: + if resource == 'users': + # We should only impose a default password if the resource doesn't exist. + post_data.setdefault('password', 'password') page = endpoint.post(post_data) else: page = page.put(post_data) From ea5b810e87ba044cf9740bc0277246f8a4da933c Mon Sep 17 00:00:00 2001 From: Jeff Bradberry Date: Wed, 11 Mar 2020 16:16:23 -0400 Subject: [PATCH 22/73] Sort the asset groups to be imported by their dependency relationships --- awxkit/awxkit/cli/resource.py | 31 ++++++++++++++++++++----------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/awxkit/awxkit/cli/resource.py b/awxkit/awxkit/cli/resource.py index ad700cc23c..c30ec6fc56 100644 --- a/awxkit/awxkit/cli/resource.py +++ b/awxkit/awxkit/cli/resource.py @@ -1,3 +1,4 @@ +import itertools import json import os import sys @@ -5,6 +6,7 @@ import sys from awxkit import api, config import awxkit.exceptions as exc from awxkit.utils import to_str +from awxkit.api.mixins import has_create from awxkit.api.pages import Page from awxkit.cli.format import FORMATTERS, format_response, add_authentication_arguments from awxkit.cli.utils import CustomRegistryMeta, cprint @@ -49,13 +51,13 @@ EXPORTABLE_RESOURCES = [ 'users', 'organizations', 'teams', - # 'credential_types', - # 'credentials', - # 'notification_templates', - # 'projects', - # 'inventory', - # 'job_templates', - # 'workflow_job_templates', + 'credential_types', + 'credentials', + 'notification_templates', + 'projects', + 'inventory', + 'job_templates', + 'workflow_job_templates', ] NATURAL_KEYS = { @@ -188,6 +190,8 @@ class Import(CustomCommand): super(Import, self).__init__(*args, **kwargs) self._natural_key = {} self._options = {} + self._resource_page = {} + self._page_resource = {} def get_by_natural_key(self, key, fetch=True): frozen_key = freeze(key) @@ -222,6 +226,9 @@ class Import(CustomCommand): def register_existing_assets(self, resource): endpoint = getattr(self.v2, resource) + self._resource_page[resource] = endpoint._create().__item_class__ + self._page_resource[self._resource_page[resource]] = resource + options = endpoint.options().json['actions']['POST'] self._options[resource] = options @@ -268,13 +275,15 @@ class Import(CustomCommand): for resource in data: self.register_existing_assets(resource) - for resource, assets in data.items(): # FIXME: do a topological sort by dependencies - self.create_assets(resource, assets) + for page_cls in itertools.chain(*has_create.page_creation_order(*self._page_resource.keys())): + resource = self._page_resource[page_cls] + cprint("importing {}".format(resource), 'red', file=client.stderr) + self.create_assets(resource, data.get(resource, [])) # FIXME: should we delete existing unpatched assets? - for resource, assets in data.items(): - self.assign_related_assets(resource, assets) + # for resource, assets in data.items(): + # self.assign_related_assets(resource, assets) return {} From 3f204659a8778cf59d32494c6720763723b6c5b2 Mon Sep 17 00:00:00 2001 From: Jeff Bradberry Date: Fri, 13 Mar 2020 10:04:54 -0400 Subject: [PATCH 23/73] Temporarily disable export of resources that may not work yet --- awxkit/awxkit/cli/resource.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/awxkit/awxkit/cli/resource.py b/awxkit/awxkit/cli/resource.py index c30ec6fc56..d404abb878 100644 --- a/awxkit/awxkit/cli/resource.py +++ b/awxkit/awxkit/cli/resource.py @@ -51,13 +51,13 @@ EXPORTABLE_RESOURCES = [ 'users', 'organizations', 'teams', - 'credential_types', - 'credentials', - 'notification_templates', - 'projects', - 'inventory', - 'job_templates', - 'workflow_job_templates', + # 'credential_types', + # 'credentials', + # 'notification_templates', + # 'projects', + # 'inventory', + # 'job_templates', + # 'workflow_job_templates', ] NATURAL_KEYS = { From 868aafb263804b176d73d216a2615e982da98d77 Mon Sep 17 00:00:00 2001 From: Jeff Bradberry Date: Fri, 13 Mar 2020 14:24:29 -0400 Subject: [PATCH 24/73] Filter out managed credential types since we cannot patch them upon import. --- awxkit/awxkit/cli/resource.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/awxkit/awxkit/cli/resource.py b/awxkit/awxkit/cli/resource.py index d404abb878..3f3e0d28e6 100644 --- a/awxkit/awxkit/cli/resource.py +++ b/awxkit/awxkit/cli/resource.py @@ -51,7 +51,7 @@ EXPORTABLE_RESOURCES = [ 'users', 'organizations', 'teams', - # 'credential_types', + 'credential_types', # 'credentials', # 'notification_templates', # 'projects', @@ -316,9 +316,14 @@ class Export(CustomCommand): results = endpoint.get(all_pages=True).results options = self.get_options(endpoint) - return [self.serialize_asset(asset, options) for asset in results] + assets = (self.serialize_asset(asset, options) for asset in results) + return [asset for asset in assets if asset is not None] def serialize_asset(self, asset, options): + # Drop any (credential_type) assets that are being managed by the Tower instance. + if asset.json.get('managed_by_tower'): + return None + fields = { key: asset[key] for key in options if key in asset.json and key not in asset.related From d20fa03034bfc19931f27aab6967eafb831e7a38 Mon Sep 17 00:00:00 2001 From: Jeff Bradberry Date: Fri, 13 Mar 2020 15:43:11 -0400 Subject: [PATCH 25/73] Create new Import.dependent_resources method that yields up resource names, even ones that aren't explicitly in the import data, in dependency order. --- awxkit/awxkit/cli/resource.py | 33 +++++++++++++++++++-------------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/awxkit/awxkit/cli/resource.py b/awxkit/awxkit/cli/resource.py index 3f3e0d28e6..f82102a3ac 100644 --- a/awxkit/awxkit/cli/resource.py +++ b/awxkit/awxkit/cli/resource.py @@ -190,8 +190,6 @@ class Import(CustomCommand): super(Import, self).__init__(*args, **kwargs) self._natural_key = {} self._options = {} - self._resource_page = {} - self._page_resource = {} def get_by_natural_key(self, key, fetch=True): frozen_key = freeze(key) @@ -200,9 +198,14 @@ class Import(CustomCommand): return self._natural_key.get(frozen_key) - def create_assets(self, resource, assets): + def create_assets(self, data, resource): + if resource not in data: + return + cprint("importing {}".format(resource), 'red', file=client.stderr) + endpoint = getattr(self.v2, resource) options = self._options[resource] + assets = data[resource] for asset in assets: post_data = {} for field, value in asset.items(): @@ -226,9 +229,6 @@ class Import(CustomCommand): def register_existing_assets(self, resource): endpoint = getattr(self.v2, resource) - self._resource_page[resource] = endpoint._create().__item_class__ - self._page_resource[self._resource_page[resource]] = resource - options = endpoint.options().json['actions']['POST'] self._options[resource] = options @@ -263,6 +263,16 @@ class Import(CustomCommand): else: self.assign_related(page, name, S) + def dependent_resources(self, data): + page_resource = {} + for resource in data: + endpoint = getattr(self.v2, resource) + page_cls = endpoint._create().__item_class__ + page_resource[page_cls] = resource + + for page_cls in itertools.chain(*has_create.page_creation_order(*page_resource.keys())): + yield page_resource[page_cls] + def handle(self, client, parser): if client.help: parser.print_help() @@ -272,15 +282,10 @@ class Import(CustomCommand): client.authenticate() self.v2 = client.v2 - for resource in data: + for resource in self.dependent_resources(data): self.register_existing_assets(resource) - - for page_cls in itertools.chain(*has_create.page_creation_order(*self._page_resource.keys())): - resource = self._page_resource[page_cls] - cprint("importing {}".format(resource), 'red', file=client.stderr) - self.create_assets(resource, data.get(resource, [])) - - # FIXME: should we delete existing unpatched assets? + self.create_assets(data, resource) + # FIXME: should we delete existing unpatched assets? # for resource, assets in data.items(): # self.assign_related_assets(resource, assets) From 55f79a45699ac5cdd9c98e9a7438fa73e5f12ead Mon Sep 17 00:00:00 2001 From: Jeff Bradberry Date: Fri, 13 Mar 2020 16:47:44 -0400 Subject: [PATCH 26/73] Fix a couple of flaws - The dependency ordering may spit out page types that weren't in the import data set. Make sure to be able to map those to resources anyway. - freeze() needs to be able to deal with nullable foreign keys. --- awxkit/awxkit/cli/resource.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/awxkit/awxkit/cli/resource.py b/awxkit/awxkit/cli/resource.py index f82102a3ac..c108427208 100644 --- a/awxkit/awxkit/cli/resource.py +++ b/awxkit/awxkit/cli/resource.py @@ -179,6 +179,8 @@ class Config(CustomCommand): def freeze(key): + if key is None: + return None return frozenset((k, freeze(v) if isinstance(v, dict) else v) for k, v in key.items()) @@ -199,9 +201,9 @@ class Import(CustomCommand): return self._natural_key.get(frozen_key) def create_assets(self, data, resource): - if resource not in data: + if resource not in data or resource not in EXPORTABLE_RESOURCES: return - cprint("importing {}".format(resource), 'red', file=client.stderr) + cprint("importing {}".format(resource), 'red', file=self.client.stderr) endpoint = getattr(self.v2, resource) options = self._options[resource] @@ -212,7 +214,8 @@ class Import(CustomCommand): if field in ('related', 'natural_key'): continue if options[field]['type'] == 'id': - post_data[field] = self.get_by_natural_key(value)['id'] + page = self.get_by_natural_key(value) + post_data[field] = page['id'] if page is not None else None else: post_data[field] = value @@ -264,13 +267,11 @@ class Import(CustomCommand): self.assign_related(page, name, S) def dependent_resources(self, data): - page_resource = {} - for resource in data: - endpoint = getattr(self.v2, resource) - page_cls = endpoint._create().__item_class__ - page_resource[page_cls] = resource + page_resource = {getattr(self.v2, resource)._create().__item_class__: resource + for resource in self.v2.json} + data_pages = [getattr(self.v2, resource)._create().__item_class__ for resource in data] - for page_cls in itertools.chain(*has_create.page_creation_order(*page_resource.keys())): + for page_cls in itertools.chain(*has_create.page_creation_order(*data_pages)): yield page_resource[page_cls] def handle(self, client, parser): @@ -280,6 +281,7 @@ class Import(CustomCommand): data = json.load(sys.stdin) client.authenticate() + self.client = client self.v2 = client.v2 for resource in self.dependent_resources(data): From ad574eb896ab34a3b4c181eb9cd9df4cecb24e0f Mon Sep 17 00:00:00 2001 From: Jeff Bradberry Date: Fri, 13 Mar 2020 16:51:17 -0400 Subject: [PATCH 27/73] Enable credential export --- awxkit/awxkit/cli/resource.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/awxkit/awxkit/cli/resource.py b/awxkit/awxkit/cli/resource.py index c108427208..737331f947 100644 --- a/awxkit/awxkit/cli/resource.py +++ b/awxkit/awxkit/cli/resource.py @@ -52,7 +52,7 @@ EXPORTABLE_RESOURCES = [ 'organizations', 'teams', 'credential_types', - # 'credentials', + 'credentials', # 'notification_templates', # 'projects', # 'inventory', From 6a9add4fe3e4749d9ea339c47d2b8c2adb100a3c Mon Sep 17 00:00:00 2001 From: Jeff Bradberry Date: Tue, 17 Mar 2020 15:33:28 -0400 Subject: [PATCH 28/73] Deal somewhat reasonably with missing fields as part of the natural key - JobTemplate.organization has recently been added, we need to support with and without - WorkflowJobTemplateNode is shortly going to get an identifier field, and we will need to support both with and without --- awxkit/awxkit/cli/resource.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/awxkit/awxkit/cli/resource.py b/awxkit/awxkit/cli/resource.py index 737331f947..35f562bb7e 100644 --- a/awxkit/awxkit/cli/resource.py +++ b/awxkit/awxkit/cli/resource.py @@ -69,7 +69,7 @@ NATURAL_KEYS = { 'notification_template': ('organization', 'name'), 'project': ('organization', 'name'), 'inventory': ('organization', 'name'), - 'job_template': ('name',), + 'job_template': ('organization', 'name'), 'workflow_job_template': ('organization', 'name'), # related resources @@ -92,9 +92,11 @@ def get_natural_key(page): natural_key[key[1:]] = get_natural_key(related_objs[0].get()) elif key in page.related: natural_key[key] = get_natural_key(page.related[key].get()) - else: + elif key in page: natural_key[key] = page[key] + if not natural_key: + return None return natural_key @@ -195,7 +197,7 @@ class Import(CustomCommand): def get_by_natural_key(self, key, fetch=True): frozen_key = freeze(key) - if frozen_key not in self._natural_key and fetch: + if frozen_key is not None and frozen_key not in self._natural_key and fetch: pass return self._natural_key.get(frozen_key) @@ -228,7 +230,12 @@ class Import(CustomCommand): else: page = page.put(post_data) - self._natural_key[freeze(get_natural_key(page))] = page + self.register_page(page) + + def register_page(self, page): + natural_key = freeze(get_natural_key(page)) + if natural_key is not None: + self._natural_key[natural_key] = page def register_existing_assets(self, resource): endpoint = getattr(self.v2, resource) @@ -236,8 +243,8 @@ class Import(CustomCommand): self._options[resource] = options results = endpoint.get(all_pages=True).results - for asset in results: - self._natural_key[freeze(get_natural_key(asset))] = asset + for page in results: + self.register_page(page) def assign_roles(self, page, roles): role_endpoint = page.json['related']['roles'] @@ -260,6 +267,7 @@ class Import(CustomCommand): def assign_related_assets(self, resource, assets): for asset in assets: page = self.get_by_natural_key(asset['natural_key']) + # FIXME: deal with `page is None` case for name, S in asset.get('related', {}).items(): if name == 'roles': self.assign_roles(page, S) From 19c92a705543ef1316da61e9cff44422c187a61a Mon Sep 17 00:00:00 2001 From: Jeff Bradberry Date: Tue, 17 Mar 2020 16:38:48 -0400 Subject: [PATCH 29/73] Enable notification templates --- awxkit/awxkit/cli/resource.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/awxkit/awxkit/cli/resource.py b/awxkit/awxkit/cli/resource.py index 35f562bb7e..d7ebe698d0 100644 --- a/awxkit/awxkit/cli/resource.py +++ b/awxkit/awxkit/cli/resource.py @@ -53,7 +53,7 @@ EXPORTABLE_RESOURCES = [ 'teams', 'credential_types', 'credentials', - # 'notification_templates', + 'notification_templates', # 'projects', # 'inventory', # 'job_templates', From e4146e9bc72beec6e4147c1590b9bfe983aeb895 Mon Sep 17 00:00:00 2001 From: Jeff Bradberry Date: Fri, 20 Mar 2020 14:02:12 -0400 Subject: [PATCH 30/73] Move the export logic onto methods on the ApiV2 class making it easier to invoke programmatically. --- awxkit/awxkit/api/pages/api.py | 116 ++++++++++++++++++++++++++++++++- awxkit/awxkit/cli/resource.py | 116 ++------------------------------- 2 files changed, 119 insertions(+), 113 deletions(-) diff --git a/awxkit/awxkit/api/pages/api.py b/awxkit/awxkit/api/pages/api.py index 40348ec5d7..323afbd44a 100644 --- a/awxkit/awxkit/api/pages/api.py +++ b/awxkit/awxkit/api/pages/api.py @@ -3,6 +3,60 @@ from . import base from . import page +EXPORTABLE_RESOURCES = [ + 'users', + 'organizations', + 'teams', + 'credential_types', + 'credentials', + 'notification_templates', + # 'projects', + # 'inventory', + # 'job_templates', + # 'workflow_job_templates', +] + + +NATURAL_KEYS = { + 'user': ('username',), + 'organization': ('name',), + 'team': ('organization', 'name'), + 'credential_type': ('name', 'kind'), + 'credential': ('organization', 'name', 'credential_type'), + 'notification_template': ('organization', 'name'), + 'project': ('organization', 'name'), + 'inventory': ('organization', 'name'), + 'job_template': ('organization', 'name'), + 'workflow_job_template': ('organization', 'name'), + + # related resources + 'role': ('name', ':content_object'), +} + + +def get_natural_key(page): + natural_key = {'type': page['type']} + lookup = NATURAL_KEYS.get(page['type'], ()) + + for key in lookup or (): + if key.startswith(':'): + # treat it like a special-case related object + related_objs = [ + related for name, related in page.related.items() + if name not in ('users', 'teams') + ] + if related_objs: + natural_key[key[1:]] = get_natural_key(related_objs[0].get()) + elif key in page.related: + natural_key[key] = get_natural_key(page.related[key].get()) + elif key in page: + natural_key[key] = page[key] + + if not natural_key: + return None + return natural_key + + class Api(base.Base): pass @@ -13,7 +67,67 @@ page.register_page(resources.api, Api) class ApiV2(base.Base): - pass + def _get_options(self, endpoint): + return endpoint.options().json['actions']['POST'] + + def _serialize_asset(self, asset, options): + # Drop any (credential_type) assets that are being managed by the Tower instance. + if asset.json.get('managed_by_tower'): + return None + + fields = { + key: asset[key] for key in options + if key in asset.json and key not in asset.related + } + fields['natural_key'] = get_natural_key(asset) + + fk_fields = { + key: get_natural_key(asset.related[key].get()) for key in options + if key in asset.related + } + + related = {} + for k, related_endpoint in asset.related.items(): + if k != 'roles': + continue + data = related_endpoint.get(all_pages=True) + if 'results' in data: + related[k] = [get_natural_key(x) for x in data.results] + + related_fields = {'related': related} if related else {} + + fields.update(fk_fields) + fields.update(related_fields) + return fields + + def _get_assets(self, resource, value): + endpoint = getattr(self, resource) + if value: + from awxkit.cli.options import pk_or_name + + pk = pk_or_name(self, resource, value) + results = endpoint.get(id=pk).results + else: + results = endpoint.get(all_pages=True).results + + options = self._get_options(endpoint) + assets = (self._serialize_asset(asset, options) for asset in results) + return [asset for asset in assets if asset is not None] + + def export_assets(self, **kwargs): + # If no resource kwargs are explicitly used, export everything. + all_resources = all(kwargs.get(resource) is None for resource in EXPORTABLE_RESOURCES) + + data = {} + for resource in EXPORTABLE_RESOURCES: + value = kwargs.get(resource) + if all_resources or value is not None: + data[resource] = self._get_assets(resource, value) + + return data + + def import_assets(self): + pass page.register_page(resources.v2, ApiV2) diff --git a/awxkit/awxkit/cli/resource.py b/awxkit/awxkit/cli/resource.py index d7ebe698d0..88a3118718 100644 --- a/awxkit/awxkit/cli/resource.py +++ b/awxkit/awxkit/cli/resource.py @@ -8,6 +8,7 @@ import awxkit.exceptions as exc from awxkit.utils import to_str from awxkit.api.mixins import has_create from awxkit.api.pages import Page +from awxkit.api.pages.api import EXPORTABLE_RESOURCES, get_natural_key from awxkit.cli.format import FORMATTERS, format_response, add_authentication_arguments from awxkit.cli.utils import CustomRegistryMeta, cprint @@ -47,58 +48,6 @@ DEPRECATED_RESOURCES_REVERSE = dict( (v, k) for k, v in DEPRECATED_RESOURCES.items() ) -EXPORTABLE_RESOURCES = [ - 'users', - 'organizations', - 'teams', - 'credential_types', - 'credentials', - 'notification_templates', - # 'projects', - # 'inventory', - # 'job_templates', - # 'workflow_job_templates', -] - -NATURAL_KEYS = { - 'user': ('username',), - 'organization': ('name',), - 'team': ('organization', 'name'), - 'credential_type': ('name', 'kind'), - 'credential': ('organization', 'name', 'credential_type'), - 'notification_template': ('organization', 'name'), - 'project': ('organization', 'name'), - 'inventory': ('organization', 'name'), - 'job_template': ('organization', 'name'), - 'workflow_job_template': ('organization', 'name'), - - # related resources - 'role': ('name', ':content_object'), -} - - -def get_natural_key(page): - natural_key = {'type': page['type']} - lookup = NATURAL_KEYS.get(page['type'], ()) - - for key in lookup or (): - if key.startswith(':'): - # treat it like a special-case related object - related_objs = [ - related for name, related in page.related.items() - if name not in ('users', 'teams') - ] - if related_objs: - natural_key[key[1:]] = get_natural_key(related_objs[0].get()) - elif key in page.related: - natural_key[key] = get_natural_key(page.related[key].get()) - elif key in page: - natural_key[key] = page[key] - - if not natural_key: - return None - return natural_key - class CustomCommand(metaclass=CustomRegistryMeta): """Base class for implementing custom commands. @@ -317,53 +266,6 @@ class Export(CustomCommand): # 3) the resource flag is used with an argument, and the attr will be that argument's value resources.add_argument('--{}'.format(resource), nargs='?', const='') - def get_options(self, endpoint): - return endpoint.options().json['actions']['POST'] - - def get_assets(self, resource, value): - endpoint = getattr(self.v2, resource) - if value: - from .options import pk_or_name - - pk = pk_or_name(self.v2, resource, value) - results = endpoint.get(id=pk).results - else: - results = endpoint.get(all_pages=True).results - - options = self.get_options(endpoint) - assets = (self.serialize_asset(asset, options) for asset in results) - return [asset for asset in assets if asset is not None] - - def serialize_asset(self, asset, options): - # Drop any (credential_type) assets that are being managed by the Tower instance. - if asset.json.get('managed_by_tower'): - return None - - fields = { - key: asset[key] for key in options - if key in asset.json and key not in asset.related - } - fields['natural_key'] = get_natural_key(asset) - - fk_fields = { - key: get_natural_key(asset.related[key].get()) for key in options - if key in asset.related - } - - related = {} - for k, related_endpoint in asset.related.items(): - if k != 'roles': - continue - data = related_endpoint.get(all_pages=True) - if 'results' in data: - related[k] = [get_natural_key(x) for x in data.results] - - related_fields = {'related': related} if related else {} - - fields.update(fk_fields) - fields.update(related_fields) - return fields - def handle(self, client, parser): self.extend_parser(parser) @@ -371,21 +273,11 @@ class Export(CustomCommand): parser.print_help() raise SystemExit() - client.authenticate() parsed = parser.parse_known_args()[0] + kwargs = {resource: getattr(parsed, resource, None) for resource in EXPORTABLE_RESOURCES} - # If no resource flags are explicitly used, export everything. - all_resources = all(getattr(parsed, resource, None) is None for resource in EXPORTABLE_RESOURCES) - - self.v2 = client.v2 - - data = {} - for resource in EXPORTABLE_RESOURCES: - value = getattr(parsed, resource, None) - if all_resources or value is not None: - data[resource] = self.get_assets(resource, value) - - return data + client.authenticate() + return client.v2.export_assets(**kwargs) def parse_resource(client, skip_deprecated=False): From 3860c7597fb87675592ddaf7bb2f73ad15d395de Mon Sep 17 00:00:00 2001 From: Jeff Bradberry Date: Fri, 20 Mar 2020 14:33:31 -0400 Subject: [PATCH 31/73] Move the import logic into methods on the ApiV2 class --- awxkit/awxkit/api/pages/api.py | 127 ++++++++++++++++++++++++++++++--- awxkit/awxkit/cli/resource.py | 115 +---------------------------- 2 files changed, 120 insertions(+), 122 deletions(-) diff --git a/awxkit/awxkit/api/pages/api.py b/awxkit/awxkit/api/pages/api.py index 323afbd44a..ec1e246707 100644 --- a/awxkit/awxkit/api/pages/api.py +++ b/awxkit/awxkit/api/pages/api.py @@ -1,6 +1,10 @@ +import itertools + from awxkit.api.resources import resources +import awxkit.exceptions as exc from . import base from . import page +from ..mixins import has_create EXPORTABLE_RESOURCES = [ @@ -34,29 +38,35 @@ NATURAL_KEYS = { } -def get_natural_key(page): - natural_key = {'type': page['type']} - lookup = NATURAL_KEYS.get(page['type'], ()) +def get_natural_key(pg): + natural_key = {'type': pg['type']} + lookup = NATURAL_KEYS.get(pg['type'], ()) for key in lookup or (): if key.startswith(':'): # treat it like a special-case related object related_objs = [ - related for name, related in page.related.items() + related for name, related in pg.related.items() if name not in ('users', 'teams') ] if related_objs: natural_key[key[1:]] = get_natural_key(related_objs[0].get()) - elif key in page.related: - natural_key[key] = get_natural_key(page.related[key].get()) - elif key in page: - natural_key[key] = page[key] + elif key in pg.related: + natural_key[key] = get_natural_key(pg.related[key].get()) + elif key in pg: + natural_key[key] = pg[key] if not natural_key: return None return natural_key +def freeze(key): + if key is None: + return None + return frozenset((k, freeze(v) if isinstance(v, dict) else v) for k, v in key.items()) + + class Api(base.Base): pass @@ -114,6 +124,97 @@ class ApiV2(base.Base): assets = (self._serialize_asset(asset, options) for asset in results) return [asset for asset in assets if asset is not None] + def _dependent_resources(self, data): + page_resource = {getattr(self, resource)._create().__item_class__: resource + for resource in self.json} + data_pages = [getattr(self, resource)._create().__item_class__ for resource in data] + + for page_cls in itertools.chain(*has_create.page_creation_order(*data_pages)): + yield page_resource[page_cls] + + def _register_page(self, page): + natural_key = freeze(get_natural_key(page)) + if natural_key is not None: + if getattr(self, '_natural_key', None) is None: + self._natural_key = {} + + self._natural_key[natural_key] = page + + def _register_existing_assets(self, resource): + endpoint = getattr(self, resource) + options = self._get_options(endpoint) + if getattr(self, '_options', None) is None: + self._options = {} + self._options[resource] = options + + results = endpoint.get(all_pages=True).results + for pg in results: + self._register_page(pg) + + def _get_by_natural_key(self, key, fetch=True): + frozen_key = freeze(key) + if frozen_key is not None and frozen_key not in self._natural_key and fetch: + pass # FIXME + + return self._natural_key.get(frozen_key) + + def _create_assets(self, data, resource): + if resource not in data or resource not in EXPORTABLE_RESOURCES: + return + + endpoint = getattr(self, resource) + options = self._options[resource] + assets = data[resource] + for asset in assets: + post_data = {} + for field, value in asset.items(): + if field not in options: + continue + if options[field]['type'] == 'id': + page = self._get_by_natural_key(value) + post_data[field] = page['id'] if page is not None else None + else: + post_data[field] = value + + page = self._get_by_natural_key(asset['natural_key'], fetch=False) + if page is None: + if resource == 'users': + # We should only impose a default password if the resource doesn't exist. + post_data.setdefault('password', 'abc123') + page = endpoint.post(post_data) + else: + page = page.put(post_data) + + self._register_page(page) + + def _assign_related(self, page, name, related_set): + pass # FIXME + + def _assign_roles(self, page, roles): + role_endpoint = page.json['related']['roles'] + for role in roles: + if 'content_object' not in role: + continue # admin role + obj_page = self._get_by_natural_key(role['content_object']) + if obj_page is not None: + role_page = obj_page.get_object_role(role['name'], by_name=True) + try: + role_endpoint.post({'id': role_page['id']}) + except exc.NoContent: # desired exception on successful (dis)association + pass + else: + pass # admin role + + def assign_related_assets(self, resource, assets): + for asset in assets: + page = self._get_by_natural_key(asset['natural_key']) + # FIXME: deal with `page is None` case + for name, S in asset.get('related', {}).items(): + if name == 'roles': + self._assign_roles(page, S) + else: + self._assign_related(page, name, S) + def export_assets(self, **kwargs): # If no resource kwargs are explicitly used, export everything. all_resources = all(kwargs.get(resource) is None for resource in EXPORTABLE_RESOURCES) @@ -126,8 +227,14 @@ class ApiV2(base.Base): return data - def import_assets(self): - pass + def import_assets(self, data): + for resource in self._dependent_resources(data): + self._register_existing_assets(resource) + self._create_assets(data, resource) + # FIXME: should we delete existing unpatched assets? + + # for resource, assets in data.items(): + # self.assign_related_assets(resource, assets) page.register_page(resources.v2, ApiV2) diff --git a/awxkit/awxkit/cli/resource.py b/awxkit/awxkit/cli/resource.py index 88a3118718..f22795fab2 100644 --- a/awxkit/awxkit/cli/resource.py +++ b/awxkit/awxkit/cli/resource.py @@ -1,14 +1,11 @@ -import itertools import json import os import sys from awxkit import api, config -import awxkit.exceptions as exc from awxkit.utils import to_str -from awxkit.api.mixins import has_create from awxkit.api.pages import Page -from awxkit.api.pages.api import EXPORTABLE_RESOURCES, get_natural_key +from awxkit.api.pages.api import EXPORTABLE_RESOURCES from awxkit.cli.format import FORMATTERS, format_response, add_authentication_arguments from awxkit.cli.utils import CustomRegistryMeta, cprint @@ -129,125 +126,19 @@ class Config(CustomCommand): } -def freeze(key): - if key is None: - return None - return frozenset((k, freeze(v) if isinstance(v, dict) else v) for k, v in key.items()) - - class Import(CustomCommand): name = 'import' help_text = 'import resources into Tower' - def __init__(self, *args, **kwargs): - super(Import, self).__init__(*args, **kwargs) - self._natural_key = {} - self._options = {} - - def get_by_natural_key(self, key, fetch=True): - frozen_key = freeze(key) - if frozen_key is not None and frozen_key not in self._natural_key and fetch: - pass - - return self._natural_key.get(frozen_key) - - def create_assets(self, data, resource): - if resource not in data or resource not in EXPORTABLE_RESOURCES: - return - cprint("importing {}".format(resource), 'red', file=self.client.stderr) - - endpoint = getattr(self.v2, resource) - options = self._options[resource] - assets = data[resource] - for asset in assets: - post_data = {} - for field, value in asset.items(): - if field in ('related', 'natural_key'): - continue - if options[field]['type'] == 'id': - page = self.get_by_natural_key(value) - post_data[field] = page['id'] if page is not None else None - else: - post_data[field] = value - - page = self.get_by_natural_key(asset['natural_key'], fetch=False) - if page is None: - if resource == 'users': - # We should only impose a default password if the resource doesn't exist. - post_data.setdefault('password', 'password') - page = endpoint.post(post_data) - else: - page = page.put(post_data) - - self.register_page(page) - - def register_page(self, page): - natural_key = freeze(get_natural_key(page)) - if natural_key is not None: - self._natural_key[natural_key] = page - - def register_existing_assets(self, resource): - endpoint = getattr(self.v2, resource) - options = endpoint.options().json['actions']['POST'] - self._options[resource] = options - - results = endpoint.get(all_pages=True).results - for page in results: - self.register_page(page) - - def assign_roles(self, page, roles): - role_endpoint = page.json['related']['roles'] - for role in roles: - if 'content_object' not in role: - continue # admin role - obj_page = self.get_by_natural_key(role['content_object']) - if obj_page is not None: - role_page = obj_page.get_object_role(role['name'], by_name=True) - try: - role_endpoint.post({'id': role_page['id']}) - except exc.NoContent: # desired exception on successful (dis)association - pass - else: - pass # admin role - - def assign_related(self, page, name, related_set): - pass - - def assign_related_assets(self, resource, assets): - for asset in assets: - page = self.get_by_natural_key(asset['natural_key']) - # FIXME: deal with `page is None` case - for name, S in asset.get('related', {}).items(): - if name == 'roles': - self.assign_roles(page, S) - else: - self.assign_related(page, name, S) - - def dependent_resources(self, data): - page_resource = {getattr(self.v2, resource)._create().__item_class__: resource - for resource in self.v2.json} - data_pages = [getattr(self.v2, resource)._create().__item_class__ for resource in data] - - for page_cls in itertools.chain(*has_create.page_creation_order(*data_pages)): - yield page_resource[page_cls] - def handle(self, client, parser): if client.help: parser.print_help() raise SystemExit() data = json.load(sys.stdin) + client.authenticate() - self.client = client - self.v2 = client.v2 - - for resource in self.dependent_resources(data): - self.register_existing_assets(resource) - self.create_assets(data, resource) - # FIXME: should we delete existing unpatched assets? - - # for resource, assets in data.items(): - # self.assign_related_assets(resource, assets) + client.v2.import_assets(data) return {} From e0f6af47001b38e0a615960551bdfe7328c9e2e8 Mon Sep 17 00:00:00 2001 From: Jeff Bradberry Date: Mon, 23 Mar 2020 16:42:34 -0400 Subject: [PATCH 32/73] Enable the export of several relation field types --- awxkit/awxkit/api/pages/api.py | 27 +++++++++++++++++++++++---- awxkit/awxkit/api/pages/page.py | 2 +- 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/awxkit/awxkit/api/pages/api.py b/awxkit/awxkit/api/pages/api.py index ec1e246707..92295dd8f6 100644 --- a/awxkit/awxkit/api/pages/api.py +++ b/awxkit/awxkit/api/pages/api.py @@ -21,6 +21,15 @@ EXPORTABLE_RESOURCES = [ ] +EXPORTABLE_RELATIONS = [ + 'Roles', + 'NotificationTemplates', + 'Labels', + 'SurveySpec', + 'WorkflowJobTemplateNodes', +] + + NATURAL_KEYS = { 'user': ('username',), 'organization': ('name',), @@ -35,6 +44,9 @@ NATURAL_KEYS = { # related resources 'role': ('name', ':content_object'), + 'notification_template': ('organization', 'name'), + 'label': ('organization', 'name'), # FIXME: label will need to be fully constructed from this + 'workflow_job_template_node': ('workflow_job_template', 'identifier'), } @@ -98,11 +110,18 @@ class ApiV2(base.Base): related = {} for k, related_endpoint in asset.related.items(): - if k != 'roles': + if not related_endpoint: + continue + if k == 'object_roles': + continue + rel = related_endpoint._create() + if rel.__class__.__name__ not in EXPORTABLE_RELATIONS: continue data = related_endpoint.get(all_pages=True) if 'results' in data: related[k] = [get_natural_key(x) for x in data.results] + else: + related[k] = data.json related_fields = {'related': related} if related else {} @@ -115,7 +134,7 @@ class ApiV2(base.Base): if value: from awxkit.cli.options import pk_or_name - pk = pk_or_name(self, resource, value) + pk = pk_or_name(self, resource, value) # TODO: decide whether to support multiple results = endpoint.get(id=pk).results else: results = endpoint.get(all_pages=True).results @@ -205,7 +224,7 @@ class ApiV2(base.Base): else: pass # admin role - def assign_related_assets(self, resource, assets): + def _assign_related_assets(self, resource, assets): for asset in assets: page = self._get_by_natural_key(asset['natural_key']) # FIXME: deal with `page is None` case @@ -234,7 +253,7 @@ class ApiV2(base.Base): # FIXME: should we delete existing unpatched assets? # for resource, assets in data.items(): - # self.assign_related_assets(resource, assets) + # self._assign_related_assets(resource, assets) page.register_page(resources.v2, ApiV2) diff --git a/awxkit/awxkit/api/pages/page.py b/awxkit/awxkit/api/pages/page.py index 7e714bbb05..256424a214 100644 --- a/awxkit/awxkit/api/pages/page.py +++ b/awxkit/awxkit/api/pages/page.py @@ -273,7 +273,7 @@ class Page(object): def get(self, all_pages=False, **query_parameters): r = self.connection.get(self.endpoint, query_parameters) page = self.page_identity(r) - if all_pages and page.next: + if all_pages and getattr(page, 'next', None): paged_results = [r.json()['results']] while page.next: r = self.connection.get(self.next, query_parameters) From 01d575f833b63291d817997beb8d4a6bbfcd7aa9 Mon Sep 17 00:00:00 2001 From: Jeff Bradberry Date: Tue, 24 Mar 2020 15:56:40 -0400 Subject: [PATCH 33/73] Enable the remaining top-level exportable resources and rearrange the import/export methods a bit. --- awxkit/awxkit/api/pages/api.py | 50 ++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 21 deletions(-) diff --git a/awxkit/awxkit/api/pages/api.py b/awxkit/awxkit/api/pages/api.py index 92295dd8f6..ca93782e10 100644 --- a/awxkit/awxkit/api/pages/api.py +++ b/awxkit/awxkit/api/pages/api.py @@ -14,10 +14,10 @@ EXPORTABLE_RESOURCES = [ 'credential_types', 'credentials', 'notification_templates', - # 'projects', - # 'inventory', - # 'job_templates', - # 'workflow_job_templates', + 'projects', + 'inventory', + 'job_templates', + 'workflow_job_templates', ] @@ -89,9 +89,13 @@ page.register_page(resources.api, Api) class ApiV2(base.Base): + # Common import/export methods + def _get_options(self, endpoint): return endpoint.options().json['actions']['POST'] + # Export methods + def _serialize_asset(self, asset, options): # Drop any (credential_type) assets that are being managed by the Tower instance. if asset.json.get('managed_by_tower'): @@ -143,6 +147,20 @@ class ApiV2(base.Base): assets = (self._serialize_asset(asset, options) for asset in results) return [asset for asset in assets if asset is not None] + def export_assets(self, **kwargs): + # If no resource kwargs are explicitly used, export everything. + all_resources = all(kwargs.get(resource) is None for resource in EXPORTABLE_RESOURCES) + + data = {} + for resource in EXPORTABLE_RESOURCES: + value = kwargs.get(resource) + if all_resources or value is not None: + data[resource] = self._get_assets(resource, value) + + return data + + # Import methods + def _dependent_resources(self, data): page_resource = {getattr(self, resource)._create().__item_class__: resource for resource in self.json} @@ -153,6 +171,8 @@ class ApiV2(base.Base): def _register_page(self, page): natural_key = freeze(get_natural_key(page)) + # FIXME: we need to keep a reference for the case where we + # don't have a natural key, so we can delete if natural_key is not None: if getattr(self, '_natural_key', None) is None: self._natural_key = {} @@ -206,9 +226,6 @@ class ApiV2(base.Base): self._register_page(page) - def _assign_related(self, page, name, related_set): - pass # FIXME - def _assign_roles(self, page, roles): role_endpoint = page.json['related']['roles'] for role in roles: @@ -224,6 +241,9 @@ class ApiV2(base.Base): else: pass # admin role + def _assign_related(self, page, name, related_set): + pass # FIXME + def _assign_related_assets(self, resource, assets): for asset in assets: page = self._get_by_natural_key(asset['natural_key']) @@ -234,26 +254,14 @@ class ApiV2(base.Base): else: self._assign_related(page, name, S) - def export_assets(self, **kwargs): - # If no resource kwargs are explicitly used, export everything. - all_resources = all(kwargs.get(resource) is None for resource in EXPORTABLE_RESOURCES) - - data = {} - for resource in EXPORTABLE_RESOURCES: - value = kwargs.get(resource) - if all_resources or value is not None: - data[resource] = self._get_assets(resource, value) - - return data - def import_assets(self, data): for resource in self._dependent_resources(data): self._register_existing_assets(resource) self._create_assets(data, resource) # FIXME: should we delete existing unpatched assets? - # for resource, assets in data.items(): - # self._assign_related_assets(resource, assets) + for resource, assets in data.items(): + self._assign_related_assets(resource, assets) page.register_page(resources.v2, ApiV2) From 30a3e3e1720d6b8819feb4d0f21ebb3d43eae04d Mon Sep 17 00:00:00 2001 From: Jeff Bradberry Date: Tue, 24 Mar 2020 16:40:23 -0400 Subject: [PATCH 34/73] Deal with lack of permissions --- awxkit/awxkit/api/pages/api.py | 51 ++++++++++++++++++---------------- 1 file changed, 27 insertions(+), 24 deletions(-) diff --git a/awxkit/awxkit/api/pages/api.py b/awxkit/awxkit/api/pages/api.py index ca93782e10..e1bf098eae 100644 --- a/awxkit/awxkit/api/pages/api.py +++ b/awxkit/awxkit/api/pages/api.py @@ -92,7 +92,7 @@ class ApiV2(base.Base): # Common import/export methods def _get_options(self, endpoint): - return endpoint.options().json['actions']['POST'] + return endpoint.options().json['actions'].get('POST', {}) # Export methods @@ -101,31 +101,34 @@ class ApiV2(base.Base): if asset.json.get('managed_by_tower'): return None - fields = { - key: asset[key] for key in options - if key in asset.json and key not in asset.related - } - fields['natural_key'] = get_natural_key(asset) + try: + fields = { + key: asset[key] for key in options + if key in asset.json and key not in asset.related + } + fields['natural_key'] = get_natural_key(asset) - fk_fields = { - key: get_natural_key(asset.related[key].get()) for key in options - if key in asset.related - } + fk_fields = { + key: get_natural_key(asset.related[key].get()) for key in options + if key in asset.related + } - related = {} - for k, related_endpoint in asset.related.items(): - if not related_endpoint: - continue - if k == 'object_roles': - continue - rel = related_endpoint._create() - if rel.__class__.__name__ not in EXPORTABLE_RELATIONS: - continue - data = related_endpoint.get(all_pages=True) - if 'results' in data: - related[k] = [get_natural_key(x) for x in data.results] - else: - related[k] = data.json + related = {} + for k, related_endpoint in asset.related.items(): + if not related_endpoint: + continue + if k == 'object_roles': + continue + rel = related_endpoint._create() + if rel.__class__.__name__ not in EXPORTABLE_RELATIONS: + continue + data = related_endpoint.get(all_pages=True) + if 'results' in data: + related[k] = [get_natural_key(x) for x in data.results] + else: + related[k] = data.json + except exc.Forbidden: + return None related_fields = {'related': related} if related else {} From a1f7d0b78163fa231e7529bc288be7545c0792ca Mon Sep 17 00:00:00 2001 From: Jeff Bradberry Date: Fri, 27 Mar 2020 11:06:32 -0400 Subject: [PATCH 35/73] Fix a bug with inventory.variables Using Page[key] instead of Page.json[key] causes inner json blob strings to automatically get parsed, which is not what we want with this field. --- awxkit/awxkit/api/pages/api.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/awxkit/awxkit/api/pages/api.py b/awxkit/awxkit/api/pages/api.py index e1bf098eae..2bb63df3b1 100644 --- a/awxkit/awxkit/api/pages/api.py +++ b/awxkit/awxkit/api/pages/api.py @@ -102,8 +102,9 @@ class ApiV2(base.Base): return None try: + # Note: doing asset[key] automatically parses json blob strings, which can be a problem. fields = { - key: asset[key] for key in options + key: asset.json[key] for key in options if key in asset.json and key not in asset.related } fields['natural_key'] = get_natural_key(asset) From 972d3ab53551606acea06a9d0d1ed39499b974ad Mon Sep 17 00:00:00 2001 From: Jeff Bradberry Date: Fri, 27 Mar 2020 16:08:45 -0400 Subject: [PATCH 36/73] Export full related objects under some conditions --- awxkit/awxkit/api/pages/api.py | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/awxkit/awxkit/api/pages/api.py b/awxkit/awxkit/api/pages/api.py index 2bb63df3b1..342c36c56c 100644 --- a/awxkit/awxkit/api/pages/api.py +++ b/awxkit/awxkit/api/pages/api.py @@ -24,6 +24,10 @@ EXPORTABLE_RESOURCES = [ EXPORTABLE_RELATIONS = [ 'Roles', 'NotificationTemplates', +] + + +EXPORTABLE_DEPENDENT_OBJECTS = [ 'Labels', 'SurveySpec', 'WorkflowJobTemplateNodes', @@ -105,7 +109,7 @@ class ApiV2(base.Base): # Note: doing asset[key] automatically parses json blob strings, which can be a problem. fields = { key: asset.json[key] for key in options - if key in asset.json and key not in asset.related + if key in asset.json and key not in asset.related and key != 'id' } fields['natural_key'] = get_natural_key(asset) @@ -115,19 +119,29 @@ class ApiV2(base.Base): } related = {} - for k, related_endpoint in asset.related.items(): - if not related_endpoint: + for key, related_endpoint in asset.related.items(): + if key in asset.json or not related_endpoint: continue - if k == 'object_roles': + if key == 'object_roles': continue rel = related_endpoint._create() - if rel.__class__.__name__ not in EXPORTABLE_RELATIONS: + + if rel.__class__.__name__ in EXPORTABLE_RELATIONS: + by_natural_key = True + elif rel.__class__.__name__ in EXPORTABLE_DEPENDENT_OBJECTS: + by_natural_key = False + else: continue + data = related_endpoint.get(all_pages=True) if 'results' in data: - related[k] = [get_natural_key(x) for x in data.results] + related_options = self._get_options(related_endpoint) + related[key] = [ + get_natural_key(x) if by_natural_key else self._serialize_asset(x, related_options) + for x in data.results + ] else: - related[k] = data.json + related[key] = data.json except exc.Forbidden: return None From 07ba521b8bc959f2d7a9aab7f368f4c35da8877e Mon Sep 17 00:00:00 2001 From: Jeff Bradberry Date: Fri, 27 Mar 2020 16:29:31 -0400 Subject: [PATCH 37/73] Enable schedules --- awxkit/awxkit/api/pages/api.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/awxkit/awxkit/api/pages/api.py b/awxkit/awxkit/api/pages/api.py index 342c36c56c..8b45213459 100644 --- a/awxkit/awxkit/api/pages/api.py +++ b/awxkit/awxkit/api/pages/api.py @@ -31,6 +31,7 @@ EXPORTABLE_DEPENDENT_OBJECTS = [ 'Labels', 'SurveySpec', 'WorkflowJobTemplateNodes', + 'Schedules', ] @@ -51,6 +52,7 @@ NATURAL_KEYS = { 'notification_template': ('organization', 'name'), 'label': ('organization', 'name'), # FIXME: label will need to be fully constructed from this 'workflow_job_template_node': ('workflow_job_template', 'identifier'), + 'schedule': ('unified_job_template', 'name'), } From e92c8cfdccee15e19e4c47a55749414d978713cb Mon Sep 17 00:00:00 2001 From: Jeff Bradberry Date: Mon, 30 Mar 2020 16:07:12 -0400 Subject: [PATCH 38/73] Remove the natural key lookup dict and make each Page responsible instead --- awxkit/awxkit/api/pages/api.py | 55 ++----------------- awxkit/awxkit/api/pages/credentials.py | 3 + awxkit/awxkit/api/pages/inventory.py | 1 + awxkit/awxkit/api/pages/job_templates.py | 1 + awxkit/awxkit/api/pages/labels.py | 1 + .../api/pages/notification_templates.py | 1 + awxkit/awxkit/api/pages/organizations.py | 2 + awxkit/awxkit/api/pages/page.py | 21 +++++++ awxkit/awxkit/api/pages/projects.py | 1 + awxkit/awxkit/api/pages/roles.py | 14 ++++- awxkit/awxkit/api/pages/schedules.py | 2 +- awxkit/awxkit/api/pages/teams.py | 1 + awxkit/awxkit/api/pages/users.py | 2 + .../api/pages/workflow_job_template_nodes.py | 1 + .../api/pages/workflow_job_templates.py | 1 + awxkit/awxkit/exceptions.py | 5 ++ 16 files changed, 61 insertions(+), 51 deletions(-) diff --git a/awxkit/awxkit/api/pages/api.py b/awxkit/awxkit/api/pages/api.py index 8b45213459..a79e49d57e 100644 --- a/awxkit/awxkit/api/pages/api.py +++ b/awxkit/awxkit/api/pages/api.py @@ -35,50 +35,6 @@ EXPORTABLE_DEPENDENT_OBJECTS = [ ] -NATURAL_KEYS = { - 'user': ('username',), - 'organization': ('name',), - 'team': ('organization', 'name'), - 'credential_type': ('name', 'kind'), - 'credential': ('organization', 'name', 'credential_type'), - 'notification_template': ('organization', 'name'), - 'project': ('organization', 'name'), - 'inventory': ('organization', 'name'), - 'job_template': ('organization', 'name'), - 'workflow_job_template': ('organization', 'name'), - - # related resources - 'role': ('name', ':content_object'), - 'notification_template': ('organization', 'name'), - 'label': ('organization', 'name'), # FIXME: label will need to be fully constructed from this - 'workflow_job_template_node': ('workflow_job_template', 'identifier'), - 'schedule': ('unified_job_template', 'name'), -} - - -def get_natural_key(pg): - natural_key = {'type': pg['type']} - lookup = NATURAL_KEYS.get(pg['type'], ()) - - for key in lookup or (): - if key.startswith(':'): - # treat it like a special-case related object - related_objs = [ - related for name, related in pg.related.items() - if name not in ('users', 'teams') - ] - if related_objs: - natural_key[key[1:]] = get_natural_key(related_objs[0].get()) - elif key in pg.related: - natural_key[key] = get_natural_key(pg.related[key].get()) - elif key in pg: - natural_key[key] = pg[key] - - if not natural_key: - return None - return natural_key - - def freeze(key): if key is None: return None @@ -113,10 +69,11 @@ class ApiV2(base.Base): key: asset.json[key] for key in options if key in asset.json and key not in asset.related and key != 'id' } - fields['natural_key'] = get_natural_key(asset) + fields['natural_key'] = asset.get_natural_key() fk_fields = { - key: get_natural_key(asset.related[key].get()) for key in options + # FIXME: use caching by url + key: asset.related[key].get().get_natural_key() for key in options if key in asset.related } @@ -124,7 +81,7 @@ class ApiV2(base.Base): for key, related_endpoint in asset.related.items(): if key in asset.json or not related_endpoint: continue - if key == 'object_roles': + if key == 'object_roles': # FIXME continue rel = related_endpoint._create() @@ -139,7 +96,7 @@ class ApiV2(base.Base): if 'results' in data: related_options = self._get_options(related_endpoint) related[key] = [ - get_natural_key(x) if by_natural_key else self._serialize_asset(x, related_options) + x.get_natural_key() if by_natural_key else self._serialize_asset(x, related_options) for x in data.results ] else: @@ -190,7 +147,7 @@ class ApiV2(base.Base): yield page_resource[page_cls] def _register_page(self, page): - natural_key = freeze(get_natural_key(page)) + natural_key = freeze(page.get_natural_key()) # FIXME: we need to keep a reference for the case where we # don't have a natural key, so we can delete if natural_key is not None: diff --git a/awxkit/awxkit/api/pages/credentials.py b/awxkit/awxkit/api/pages/credentials.py index 8a5e7e0eca..88f89dd3b3 100644 --- a/awxkit/awxkit/api/pages/credentials.py +++ b/awxkit/awxkit/api/pages/credentials.py @@ -149,6 +149,8 @@ def get_payload_field_and_value_from_kwargs_or_config_cred( class CredentialType(HasCreate, base.Base): + NATURAL_KEY = ('name', 'kind') + def silent_delete(self): if not self.managed_by_tower: return super(CredentialType, self).silent_delete() @@ -204,6 +206,7 @@ class Credential(HasCopy, HasCreate, base.Base): dependencies = [CredentialType] optional_dependencies = [Organization, User, Team] + NATURAL_KEY = ('organization', 'name', 'credential_type') def payload( self, diff --git a/awxkit/awxkit/api/pages/inventory.py b/awxkit/awxkit/api/pages/inventory.py index 2b057153d3..6d89fcba3a 100644 --- a/awxkit/awxkit/api/pages/inventory.py +++ b/awxkit/awxkit/api/pages/inventory.py @@ -32,6 +32,7 @@ log = logging.getLogger(__name__) class Inventory(HasCopy, HasCreate, HasInstanceGroups, HasVariables, base.Base): dependencies = [Organization] + NATURAL_KEY = ('organization', 'name') def print_ini(self): """Print an ini version of the inventory""" diff --git a/awxkit/awxkit/api/pages/job_templates.py b/awxkit/awxkit/api/pages/job_templates.py index ad59bfb742..cd45fc0c87 100644 --- a/awxkit/awxkit/api/pages/job_templates.py +++ b/awxkit/awxkit/api/pages/job_templates.py @@ -24,6 +24,7 @@ class JobTemplate( UnifiedJobTemplate): optional_dependencies = [Inventory, Credential, Project] + NATURAL_KEY = ('organization', 'name') def launch(self, payload={}): """Launch the job_template using related->launch endpoint.""" diff --git a/awxkit/awxkit/api/pages/labels.py b/awxkit/awxkit/api/pages/labels.py index f545ef3776..a76b6920a5 100644 --- a/awxkit/awxkit/api/pages/labels.py +++ b/awxkit/awxkit/api/pages/labels.py @@ -9,6 +9,7 @@ from . import page class Label(HasCreate, base.Base): dependencies = [Organization] + NATURAL_KEY = ('organization', 'name') def silent_delete(self): """Label pages do not support DELETE requests. Here, we override the base page object diff --git a/awxkit/awxkit/api/pages/notification_templates.py b/awxkit/awxkit/api/pages/notification_templates.py index f69bb7b8a7..ff192d4433 100644 --- a/awxkit/awxkit/api/pages/notification_templates.py +++ b/awxkit/awxkit/api/pages/notification_templates.py @@ -24,6 +24,7 @@ notification_types = ( class NotificationTemplate(HasCopy, HasCreate, base.Base): dependencies = [Organization] + NATURAL_KEY = ('organization', 'name') def test(self): """Create test notification""" diff --git a/awxkit/awxkit/api/pages/organizations.py b/awxkit/awxkit/api/pages/organizations.py index cdc24083b8..413ecf4961 100644 --- a/awxkit/awxkit/api/pages/organizations.py +++ b/awxkit/awxkit/api/pages/organizations.py @@ -8,6 +8,8 @@ from . import page class Organization(HasCreate, HasInstanceGroups, HasNotifications, base.Base): + NATURAL_KEY = ('name',) + def add_admin(self, user): if isinstance(user, page.Page): user = user.json diff --git a/awxkit/awxkit/api/pages/page.py b/awxkit/awxkit/api/pages/page.py index 256424a214..9effa8ef64 100644 --- a/awxkit/awxkit/api/pages/page.py +++ b/awxkit/awxkit/api/pages/page.py @@ -317,6 +317,24 @@ class Page(object): page_cls = get_registered_page(endpoint) return page_cls(self.connection, endpoint=endpoint).get(**kw) + def get_natural_key(self): + if not getattr(self, 'NATURAL_KEY', None): + raise exc.NoNaturalKey( + "Page does not have a natural key: {}".format(getattr(self, 'endpoint', repr(self.__class__))) + ) + natural_key = {} + for key in self.NATURAL_KEY: + if key in self.related: + # FIXME: use caching by url + natural_key[key] = self.related[key].get().get_natural_key() + elif key in self: + natural_key[key] = self[key] + if not natural_key: + return None + + natural_key['type'] = self['type'] + return natural_key + _exception_map = {http.NO_CONTENT: exc.NoContent, http.NOT_FOUND: exc.NotFound, @@ -376,6 +394,9 @@ class PageList(object): def create(self, *a, **kw): return self.__item_class__(self.connection).create(*a, **kw) + def get_natural_key(self): + raise exc.NoNaturalKey + class TentativePage(str): diff --git a/awxkit/awxkit/api/pages/projects.py b/awxkit/awxkit/api/pages/projects.py index 584c151f78..e40191260c 100644 --- a/awxkit/awxkit/api/pages/projects.py +++ b/awxkit/awxkit/api/pages/projects.py @@ -14,6 +14,7 @@ class Project(HasCopy, HasCreate, HasNotifications, UnifiedJobTemplate): optional_dependencies = [Credential, Organization] optional_schedule_fields = tuple() + NATURAL_KEY = ('organization', 'name') def payload(self, organization, scm_type='git', **kwargs): payload = PseudoNamespace( diff --git a/awxkit/awxkit/api/pages/roles.py b/awxkit/awxkit/api/pages/roles.py index f8d6e39f70..a08d670840 100644 --- a/awxkit/awxkit/api/pages/roles.py +++ b/awxkit/awxkit/api/pages/roles.py @@ -5,7 +5,19 @@ from . import page class Role(base.Base): - pass + NATURAL_KEY = ('name',) + + def get_natural_key(self): + natural_key = super(Role, self).get_natural_key() + related_objs = [ + related for name, related in self.related.items() + if name not in ('users', 'teams') + ] + if related_objs: + # FIXME: use caching by url + natural_key['content_object'] = related_objs[0].get().get_natural_key() + + return natural_key page.register_page(resources.role, Role) diff --git a/awxkit/awxkit/api/pages/schedules.py b/awxkit/awxkit/api/pages/schedules.py index b0b21645eb..8603b2ad5b 100644 --- a/awxkit/awxkit/api/pages/schedules.py +++ b/awxkit/awxkit/api/pages/schedules.py @@ -8,7 +8,7 @@ from . import base class Schedule(UnifiedJob): - pass + NATURAL_KEY = ('unified_job_template', 'name') page.register_page([resources.schedule, diff --git a/awxkit/awxkit/api/pages/teams.py b/awxkit/awxkit/api/pages/teams.py index fc1e9de3f8..cb5577b5b2 100644 --- a/awxkit/awxkit/api/pages/teams.py +++ b/awxkit/awxkit/api/pages/teams.py @@ -11,6 +11,7 @@ from . import page class Team(HasCreate, base.Base): dependencies = [Organization] + NATURAL_KEY = ('organization', 'name') def add_user(self, user): if isinstance(user, page.Page): diff --git a/awxkit/awxkit/api/pages/users.py b/awxkit/awxkit/api/pages/users.py index 4039ef2e9d..22ab78dd11 100644 --- a/awxkit/awxkit/api/pages/users.py +++ b/awxkit/awxkit/api/pages/users.py @@ -9,6 +9,8 @@ from . import page class User(HasCreate, base.Base): + NATURAL_KEY = ('username',) + def payload(self, **kwargs): payload = PseudoNamespace( username=kwargs.get('username') or 'User-{}'.format( diff --git a/awxkit/awxkit/api/pages/workflow_job_template_nodes.py b/awxkit/awxkit/api/pages/workflow_job_template_nodes.py index 01e0b41e0a..1b61754928 100644 --- a/awxkit/awxkit/api/pages/workflow_job_template_nodes.py +++ b/awxkit/awxkit/api/pages/workflow_job_template_nodes.py @@ -10,6 +10,7 @@ from . import page class WorkflowJobTemplateNode(HasCreate, base.Base): dependencies = [WorkflowJobTemplate, UnifiedJobTemplate] + NATURAL_KEY = ('workflow_job_template', 'identifier') def payload(self, workflow_job_template, unified_job_template, **kwargs): if not unified_job_template: diff --git a/awxkit/awxkit/api/pages/workflow_job_templates.py b/awxkit/awxkit/api/pages/workflow_job_templates.py index 6a28891e04..17f3b56342 100644 --- a/awxkit/awxkit/api/pages/workflow_job_templates.py +++ b/awxkit/awxkit/api/pages/workflow_job_templates.py @@ -13,6 +13,7 @@ from . import page class WorkflowJobTemplate(HasCopy, HasCreate, HasNotifications, HasSurvey, UnifiedJobTemplate): optional_dependencies = [Organization] + NATURAL_KEY = ('organization', 'name') def launch(self, payload={}): """Launch using related->launch endpoint.""" diff --git a/awxkit/awxkit/exceptions.py b/awxkit/awxkit/exceptions.py index c1ff01719a..cf26097cef 100644 --- a/awxkit/awxkit/exceptions.py +++ b/awxkit/awxkit/exceptions.py @@ -101,3 +101,8 @@ class UnexpectedAWXState(Common): class IsMigrating(Common): pass + + +class NoNaturalKey(Common): + + pass From 329293dbf00d374d0e2364fb09db9c515e33ab14 Mon Sep 17 00:00:00 2001 From: Jeff Bradberry Date: Mon, 30 Mar 2020 16:21:17 -0400 Subject: [PATCH 39/73] Tentatively enable inventory sources --- awxkit/awxkit/api/pages/api.py | 1 + awxkit/awxkit/api/pages/inventory.py | 1 + 2 files changed, 2 insertions(+) diff --git a/awxkit/awxkit/api/pages/api.py b/awxkit/awxkit/api/pages/api.py index a79e49d57e..9eb0d86942 100644 --- a/awxkit/awxkit/api/pages/api.py +++ b/awxkit/awxkit/api/pages/api.py @@ -16,6 +16,7 @@ EXPORTABLE_RESOURCES = [ 'notification_templates', 'projects', 'inventory', + 'inventory_sources', 'job_templates', 'workflow_job_templates', ] diff --git a/awxkit/awxkit/api/pages/inventory.py b/awxkit/awxkit/api/pages/inventory.py index 6d89fcba3a..e00f0d329a 100644 --- a/awxkit/awxkit/api/pages/inventory.py +++ b/awxkit/awxkit/api/pages/inventory.py @@ -474,6 +474,7 @@ class InventorySource(HasCreate, HasNotifications, UnifiedJobTemplate): optional_schedule_fields = tuple() dependencies = [Inventory] optional_dependencies = [Credential, InventoryScript, Project] + NATURAL_KEY = ('organization', 'name', 'inventory') def payload( self, From 0deacc4391245662e2357934392c8f3d7ecc8ce2 Mon Sep 17 00:00:00 2001 From: Jeff Bradberry Date: Wed, 1 Apr 2020 11:32:34 -0400 Subject: [PATCH 40/73] If attempting to link to a resource that is not yet complete, wait --- awxkit/awxkit/api/pages/api.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/awxkit/awxkit/api/pages/api.py b/awxkit/awxkit/api/pages/api.py index 9eb0d86942..905aa3d152 100644 --- a/awxkit/awxkit/api/pages/api.py +++ b/awxkit/awxkit/api/pages/api.py @@ -173,7 +173,14 @@ class ApiV2(base.Base): if frozen_key is not None and frozen_key not in self._natural_key and fetch: pass # FIXME - return self._natural_key.get(frozen_key) + from awxkit.api.mixins import has_status + + _page = self._natural_key.get(frozen_key) + if isinstance(_page, has_status.HasStatus) and not _page.is_completed: + _page.wait_until_completed() + _page = _page.get() + self._natural_key[frozen_key] = _page + return _page def _create_assets(self, data, resource): if resource not in data or resource not in EXPORTABLE_RESOURCES: From 4262dd38ba697f0746888081baa06f495f175aa8 Mon Sep 17 00:00:00 2001 From: Jeff Bradberry Date: Wed, 1 Apr 2020 11:50:37 -0400 Subject: [PATCH 41/73] Change the NoNaturalKey exception to no longer derive from Common which seems to be entirely use for response exceptions. Maybe rename Common? --- awxkit/awxkit/exceptions.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/awxkit/awxkit/exceptions.py b/awxkit/awxkit/exceptions.py index cf26097cef..99d75c50d6 100644 --- a/awxkit/awxkit/exceptions.py +++ b/awxkit/awxkit/exceptions.py @@ -103,6 +103,11 @@ class IsMigrating(Common): pass -class NoNaturalKey(Common): +class ImportExportError(Exception): + + pass + + +class NoNaturalKey(ImportExportError): pass From e053a58223ea9866ee8d7f96eab2af35a0b34d44 Mon Sep 17 00:00:00 2001 From: Jeff Bradberry Date: Wed, 1 Apr 2020 15:34:42 -0400 Subject: [PATCH 42/73] Handle some more bad cases when doing OPTIONS calls - deprecated endpoints - read-only endpoints - insufficient privileges The latter case currently just drops it on the floor, but ought to do something better. --- awxkit/awxkit/api/pages/api.py | 40 +++++++++++++++++++++++++++------- 1 file changed, 32 insertions(+), 8 deletions(-) diff --git a/awxkit/awxkit/api/pages/api.py b/awxkit/awxkit/api/pages/api.py index 905aa3d152..a3c2144b5b 100644 --- a/awxkit/awxkit/api/pages/api.py +++ b/awxkit/awxkit/api/pages/api.py @@ -54,8 +54,26 @@ class ApiV2(base.Base): # Common import/export methods - def _get_options(self, endpoint): - return endpoint.options().json['actions'].get('POST', {}) + def _get_options(self, _page): + if getattr(self, '_options', None) is None: + self._options = {} + + if isinstance(_page, page.TentativePage): + url = str(_page) + else: + url = _page.url + + if url in self._options: + return self._options[url] + + options = _page.options() + warning = options.r.headers.get('Warning', '') + if '299' in warning and 'deprecated' in warning: + return self._options.setdefault(url, None) + if 'POST' not in options.r.headers.get('Allow', ''): + return self._options.setdefault(url, None) + + return self._options.setdefault(url, options.json['actions'].get('POST', {})) # Export methods @@ -63,6 +81,8 @@ class ApiV2(base.Base): # Drop any (credential_type) assets that are being managed by the Tower instance. if asset.json.get('managed_by_tower'): return None + if options is None: # Deprecated endpoint or insufficient permissions + return None try: # Note: doing asset[key] automatically parses json blob strings, which can be a problem. @@ -88,14 +108,16 @@ class ApiV2(base.Base): if rel.__class__.__name__ in EXPORTABLE_RELATIONS: by_natural_key = True + related_options = self._get_options(related_endpoint) + if related_options is None: + continue elif rel.__class__.__name__ in EXPORTABLE_DEPENDENT_OBJECTS: - by_natural_key = False + by_natural_key, related_options = False, None else: continue data = related_endpoint.get(all_pages=True) if 'results' in data: - related_options = self._get_options(related_endpoint) related[key] = [ x.get_natural_key() if by_natural_key else self._serialize_asset(x, related_options) for x in data.results @@ -113,6 +135,10 @@ class ApiV2(base.Base): def _get_assets(self, resource, value): endpoint = getattr(self, resource) + options = self._get_options(endpoint) + if options is None: + return None + if value: from awxkit.cli.options import pk_or_name @@ -121,7 +147,6 @@ class ApiV2(base.Base): else: results = endpoint.get(all_pages=True).results - options = self._get_options(endpoint) assets = (self._serialize_asset(asset, options) for asset in results) return [asset for asset in assets if asset is not None] @@ -160,9 +185,8 @@ class ApiV2(base.Base): def _register_existing_assets(self, resource): endpoint = getattr(self, resource) options = self._get_options(endpoint) - if getattr(self, '_options', None) is None: - self._options = {} - self._options[resource] = options + if options is None: + return results = endpoint.get(all_pages=True).results for pg in results: From ab15349c8c792863951763e35ecfe22543393cda Mon Sep 17 00:00:00 2001 From: Jeff Bradberry Date: Thu, 2 Apr 2020 10:49:59 -0400 Subject: [PATCH 43/73] Remove the NoNaturalKey error It's too awkward, and it makes more sense to return None instead. --- awxkit/awxkit/api/pages/page.py | 14 ++++++++------ awxkit/awxkit/api/pages/roles.py | 9 +++++++-- awxkit/awxkit/exceptions.py | 5 ----- 3 files changed, 15 insertions(+), 13 deletions(-) diff --git a/awxkit/awxkit/api/pages/page.py b/awxkit/awxkit/api/pages/page.py index 9effa8ef64..8913625b53 100644 --- a/awxkit/awxkit/api/pages/page.py +++ b/awxkit/awxkit/api/pages/page.py @@ -319,14 +319,16 @@ class Page(object): def get_natural_key(self): if not getattr(self, 'NATURAL_KEY', None): - raise exc.NoNaturalKey( - "Page does not have a natural key: {}".format(getattr(self, 'endpoint', repr(self.__class__))) - ) + return None + natural_key = {} for key in self.NATURAL_KEY: if key in self.related: - # FIXME: use caching by url - natural_key[key] = self.related[key].get().get_natural_key() + try: + # FIXME: use caching by url + natural_key[key] = self.related[key].get().get_natural_key() + except exc.Forbidden: + return None elif key in self: natural_key[key] = self[key] if not natural_key: @@ -395,7 +397,7 @@ class PageList(object): return self.__item_class__(self.connection).create(*a, **kw) def get_natural_key(self): - raise exc.NoNaturalKey + return None class TentativePage(str): diff --git a/awxkit/awxkit/api/pages/roles.py b/awxkit/awxkit/api/pages/roles.py index a08d670840..d93de4cad1 100644 --- a/awxkit/awxkit/api/pages/roles.py +++ b/awxkit/awxkit/api/pages/roles.py @@ -1,4 +1,6 @@ from awxkit.api.resources import resources +import awxkit.exceptions as exc + from . import base from . import page @@ -14,8 +16,11 @@ class Role(base.Base): if name not in ('users', 'teams') ] if related_objs: - # FIXME: use caching by url - natural_key['content_object'] = related_objs[0].get().get_natural_key() + try: + # FIXME: use caching by url + natural_key['content_object'] = related_objs[0].get().get_natural_key() + except exc.Forbidden: + return None return natural_key diff --git a/awxkit/awxkit/exceptions.py b/awxkit/awxkit/exceptions.py index 99d75c50d6..596720b59d 100644 --- a/awxkit/awxkit/exceptions.py +++ b/awxkit/awxkit/exceptions.py @@ -106,8 +106,3 @@ class IsMigrating(Common): class ImportExportError(Exception): pass - - -class NoNaturalKey(ImportExportError): - - pass From eb10a1873d550d03cc41cea8304269f64c8e2336 Mon Sep 17 00:00:00 2001 From: Jeff Bradberry Date: Thu, 2 Apr 2020 14:16:39 -0400 Subject: [PATCH 44/73] Drop objects that cannot be read or do not have a natural key don't fail hard. --- awxkit/awxkit/api/pages/api.py | 91 +++++++++++++++++++-------------- awxkit/awxkit/api/pages/page.py | 6 +++ 2 files changed, 58 insertions(+), 39 deletions(-) diff --git a/awxkit/awxkit/api/pages/api.py b/awxkit/awxkit/api/pages/api.py index a3c2144b5b..5e1a91c49c 100644 --- a/awxkit/awxkit/api/pages/api.py +++ b/awxkit/awxkit/api/pages/api.py @@ -1,4 +1,5 @@ import itertools +import logging from awxkit.api.resources import resources import awxkit.exceptions as exc @@ -7,6 +8,9 @@ from . import page from ..mixins import has_create +log = logging.getLogger(__name__) + + EXPORTABLE_RESOURCES = [ 'users', 'organizations', @@ -73,6 +77,8 @@ class ApiV2(base.Base): if 'POST' not in options.r.headers.get('Allow', ''): return self._options.setdefault(url, None) + # FIXME: if POST isn't in the actions, this is a view where we + # don't have write permissions. Try to do something anyway. return self._options.setdefault(url, options.json['actions'].get('POST', {})) # Export methods @@ -84,53 +90,60 @@ class ApiV2(base.Base): if options is None: # Deprecated endpoint or insufficient permissions return None - try: - # Note: doing asset[key] automatically parses json blob strings, which can be a problem. - fields = { - key: asset.json[key] for key in options - if key in asset.json and key not in asset.related and key != 'id' - } - fields['natural_key'] = asset.get_natural_key() + # Note: doing asset[key] automatically parses json blob strings, which can be a problem. + fields = { + key: asset.json[key] for key in options + if key in asset.json and key not in asset.related and key != 'id' + } + fields['natural_key'] = asset.get_natural_key() - fk_fields = { + for key in options: + if not key in asset.related: + continue + try: # FIXME: use caching by url - key: asset.related[key].get().get_natural_key() for key in options - if key in asset.related - } + fields[key] = asset.related[key].get().get_natural_key() + except exc.Forbidden: + log.warning("This object cannot be read: %s", asset.related[key]) + pass # FIXME: what if the fk is mandatory? - related = {} - for key, related_endpoint in asset.related.items(): - if key in asset.json or not related_endpoint: + related = {} + for key, related_endpoint in asset.related.items(): + if key in asset.json or not related_endpoint: + continue + if key == 'object_roles': + continue # FIXME: we should aggregate all visited roles + + rel = related_endpoint._create() + if rel.__class__.__name__ in EXPORTABLE_RELATIONS: + by_natural_key = True + related_options = self._get_options(related_endpoint) + if related_options is None: continue - if key == 'object_roles': # FIXME - continue - rel = related_endpoint._create() + elif rel.__class__.__name__ in EXPORTABLE_DEPENDENT_OBJECTS: + by_natural_key, related_options = False, None + else: + continue - if rel.__class__.__name__ in EXPORTABLE_RELATIONS: - by_natural_key = True - related_options = self._get_options(related_endpoint) - if related_options is None: - continue - elif rel.__class__.__name__ in EXPORTABLE_DEPENDENT_OBJECTS: - by_natural_key, related_options = False, None - else: - continue + try: + # FIXME: use caching by url + data = rel.get(all_pages=True) + except exc.Forbidden: + log.warning("This object cannot be read: %s", related_endpoint) + continue - data = related_endpoint.get(all_pages=True) - if 'results' in data: - related[key] = [ - x.get_natural_key() if by_natural_key else self._serialize_asset(x, related_options) - for x in data.results - ] - else: - related[key] = data.json - except exc.Forbidden: - return None + if 'results' in data: + results = ( + x.get_natural_key() if by_natural_key else self._serialize_asset(x, related_options) + for x in data.results + ) + related[key] = [x for x in results if x is not None] + else: + related[key] = data.json - related_fields = {'related': related} if related else {} + if related: + fields['related'] = related - fields.update(fk_fields) - fields.update(related_fields) return fields def _get_assets(self, resource, value): diff --git a/awxkit/awxkit/api/pages/page.py b/awxkit/awxkit/api/pages/page.py index 8913625b53..dd73fc4d67 100644 --- a/awxkit/awxkit/api/pages/page.py +++ b/awxkit/awxkit/api/pages/page.py @@ -318,7 +318,10 @@ class Page(object): return page_cls(self.connection, endpoint=endpoint).get(**kw) def get_natural_key(self): + warn = "This object does not have a natural key: %s" + if not getattr(self, 'NATURAL_KEY', None): + log.warning(warn, getattr(self, 'endpoint', '')) return None natural_key = {} @@ -328,10 +331,12 @@ class Page(object): # FIXME: use caching by url natural_key[key] = self.related[key].get().get_natural_key() except exc.Forbidden: + log.warning("This object cannot be read: %s", getattr(self, 'endpoint', '')) return None elif key in self: natural_key[key] = self[key] if not natural_key: + log.warning(warn, getattr(self, 'endpoint', '')) return None natural_key['type'] = self['type'] @@ -397,6 +402,7 @@ class PageList(object): return self.__item_class__(self.connection).create(*a, **kw) def get_natural_key(self): + log.warning("This object does not have a natural key: %s", getattr(self, 'endpoint', '')) return None From a5fa34bd3bd3c04436e8602bfe64e49eef2bd3ce Mon Sep 17 00:00:00 2001 From: Jeff Bradberry Date: Thu, 2 Apr 2020 15:20:43 -0400 Subject: [PATCH 45/73] Fall back to parsing the OPTIONS description to determine the needed fields for constructing an object. --- awxkit/awxkit/api/pages/api.py | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/awxkit/awxkit/api/pages/api.py b/awxkit/awxkit/api/pages/api.py index 5e1a91c49c..27d39968d2 100644 --- a/awxkit/awxkit/api/pages/api.py +++ b/awxkit/awxkit/api/pages/api.py @@ -1,5 +1,6 @@ import itertools import logging +import re from awxkit.api.resources import resources import awxkit.exceptions as exc @@ -7,6 +8,7 @@ from . import base from . import page from ..mixins import has_create +descRE = re.compile('^[*] `(\w+)`: [^(]*\((\w+), ([^)]+)\)') log = logging.getLogger(__name__) @@ -54,6 +56,17 @@ class Api(base.Base): page.register_page(resources.api, Api) +def parse_description(desc): + options = {} + for line in desc[desc.index('POST'):].splitlines(): + match = descRE.match(line) + if not match: + continue + options[match.group(1)] = {'type': match.group(2), + 'required': match.group(3) == 'required'} + return options + + class ApiV2(base.Base): # Common import/export methods @@ -77,9 +90,10 @@ class ApiV2(base.Base): if 'POST' not in options.r.headers.get('Allow', ''): return self._options.setdefault(url, None) - # FIXME: if POST isn't in the actions, this is a view where we - # don't have write permissions. Try to do something anyway. - return self._options.setdefault(url, options.json['actions'].get('POST', {})) + if 'POST' in options.json['actions']: + return self._options.setdefault(url, options.json['actions']['POST']) + else: + return self._options.setdefault(url, parse_description(options.json['description'])) # Export methods From 719f0b407c88e928b0aabcdd4094b815f5fc97ac Mon Sep 17 00:00:00 2001 From: Jeff Bradberry Date: Fri, 3 Apr 2020 15:03:54 -0400 Subject: [PATCH 46/73] Enable credential relations but only when it is in an attach/detach list view. --- awxkit/awxkit/api/pages/api.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/awxkit/awxkit/api/pages/api.py b/awxkit/awxkit/api/pages/api.py index 27d39968d2..062bad95ed 100644 --- a/awxkit/awxkit/api/pages/api.py +++ b/awxkit/awxkit/api/pages/api.py @@ -31,6 +31,7 @@ EXPORTABLE_RESOURCES = [ EXPORTABLE_RELATIONS = [ 'Roles', 'NotificationTemplates', + 'Credentials', ] @@ -125,8 +126,6 @@ class ApiV2(base.Base): for key, related_endpoint in asset.related.items(): if key in asset.json or not related_endpoint: continue - if key == 'object_roles': - continue # FIXME: we should aggregate all visited roles rel = related_endpoint._create() if rel.__class__.__name__ in EXPORTABLE_RELATIONS: @@ -134,6 +133,8 @@ class ApiV2(base.Base): related_options = self._get_options(related_endpoint) if related_options is None: continue + if 'id' not in related_options: + continue # This is a read-only or create-only endpoint. elif rel.__class__.__name__ in EXPORTABLE_DEPENDENT_OBJECTS: by_natural_key, related_options = False, None else: From 1b264011a23f1c5d4bf2ce69999b4e9c69593e84 Mon Sep 17 00:00:00 2001 From: Jeff Bradberry Date: Fri, 3 Apr 2020 16:21:26 -0400 Subject: [PATCH 47/73] Enable interconnections between WFJT Nodes --- awxkit/awxkit/api/pages/api.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/awxkit/awxkit/api/pages/api.py b/awxkit/awxkit/api/pages/api.py index 062bad95ed..e6561e5a9c 100644 --- a/awxkit/awxkit/api/pages/api.py +++ b/awxkit/awxkit/api/pages/api.py @@ -31,6 +31,7 @@ EXPORTABLE_RESOURCES = [ EXPORTABLE_RELATIONS = [ 'Roles', 'NotificationTemplates', + 'WorkflowJobTemplateNodes', 'Credentials', ] @@ -38,8 +39,10 @@ EXPORTABLE_RELATIONS = [ EXPORTABLE_DEPENDENT_OBJECTS = [ 'Labels', 'SurveySpec', - 'WorkflowJobTemplateNodes', 'Schedules', + # WFJT Nodes are a special case, we want full data for the create + # view and natural keys for the attach views. + 'WorkflowJobTemplateNodes', ] @@ -124,19 +127,19 @@ class ApiV2(base.Base): related = {} for key, related_endpoint in asset.related.items(): - if key in asset.json or not related_endpoint: + if key in options or not related_endpoint: continue rel = related_endpoint._create() - if rel.__class__.__name__ in EXPORTABLE_RELATIONS: + related_options = self._get_options(related_endpoint) + if related_options is None: # This is a read-only endpoint. + continue + is_attach = 'id' in related_options # This is not a create-only endpoint. + + if rel.__class__.__name__ in EXPORTABLE_RELATIONS and is_attach: by_natural_key = True - related_options = self._get_options(related_endpoint) - if related_options is None: - continue - if 'id' not in related_options: - continue # This is a read-only or create-only endpoint. elif rel.__class__.__name__ in EXPORTABLE_DEPENDENT_OBJECTS: - by_natural_key, related_options = False, None + by_natural_key = False else: continue From 385725e52aa492eb1f7e23fb587b0b5e4ca1f266 Mon Sep 17 00:00:00 2001 From: Jeff Bradberry Date: Mon, 6 Apr 2020 10:55:33 -0400 Subject: [PATCH 48/73] Fix the _create_assets method to use _get_options --- awxkit/awxkit/api/pages/api.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/awxkit/awxkit/api/pages/api.py b/awxkit/awxkit/api/pages/api.py index e6561e5a9c..c2ebe8e32b 100644 --- a/awxkit/awxkit/api/pages/api.py +++ b/awxkit/awxkit/api/pages/api.py @@ -8,7 +8,7 @@ from . import base from . import page from ..mixins import has_create -descRE = re.compile('^[*] `(\w+)`: [^(]*\((\w+), ([^)]+)\)') +descRE = re.compile(r'^[*] `(\w+)`: [^(]*\((\w+), ([^)]+)\)') log = logging.getLogger(__name__) @@ -116,7 +116,7 @@ class ApiV2(base.Base): fields['natural_key'] = asset.get_natural_key() for key in options: - if not key in asset.related: + if key not in asset.related: continue try: # FIXME: use caching by url @@ -242,7 +242,7 @@ class ApiV2(base.Base): return endpoint = getattr(self, resource) - options = self._options[resource] + options = self._get_options(endpoint) assets = data[resource] for asset in assets: post_data = {} From 53d81d42ccbe1825c00726ba8b0bfcfa839bb2f4 Mon Sep 17 00:00:00 2001 From: Jeff Bradberry Date: Wed, 8 Apr 2020 14:18:49 -0400 Subject: [PATCH 49/73] Hook up creation and attachment of related objects --- awxkit/awxkit/api/pages/api.py | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/awxkit/awxkit/api/pages/api.py b/awxkit/awxkit/api/pages/api.py index c2ebe8e32b..c1b2b29032 100644 --- a/awxkit/awxkit/api/pages/api.py +++ b/awxkit/awxkit/api/pages/api.py @@ -282,13 +282,33 @@ class ApiV2(base.Base): pass # admin role def _assign_related(self, page, name, related_set): - pass # FIXME + endpoint = page.related[name] + if isinstance(related_set, dict): # Relateds that are just json blobs, e.g. survey_spec + endpoint.post(related_set) + return + + if 'natural_key' not in related_set[0]: # It is an attach set + for item in related_set: + rel_page = self._get_by_natural_key(item) + if rel_page is None: + continue # FIXME + endpoint.post({'id': rel_page['id']}) + else: # It is a create set + for item in related_set: + data = {key: value for key, value in item.items() + if key not in ('natural_key', 'related')} + endpoint.post(data) + # FIXME: deal with objects that themselves have relateds, e.g. WFJT Nodes + + # FIXME: deal with pruning existing relations that do not match the import set def _assign_related_assets(self, resource, assets): for asset in assets: page = self._get_by_natural_key(asset['natural_key']) # FIXME: deal with `page is None` case for name, S in asset.get('related', {}).items(): + if not S: + continue if name == 'roles': self._assign_roles(page, S) else: From 43b76f45759ace6b26620489724d5ff63abc84d5 Mon Sep 17 00:00:00 2001 From: Jeff Bradberry Date: Wed, 8 Apr 2020 14:19:23 -0400 Subject: [PATCH 50/73] Deal with unreadable mandatory foreign keys by dropping the parent object. Also, clarify some of the warning log messages. --- awxkit/awxkit/api/pages/api.py | 9 +++++---- awxkit/awxkit/api/pages/page.py | 2 +- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/awxkit/awxkit/api/pages/api.py b/awxkit/awxkit/api/pages/api.py index c1b2b29032..f54010052d 100644 --- a/awxkit/awxkit/api/pages/api.py +++ b/awxkit/awxkit/api/pages/api.py @@ -113,7 +113,6 @@ class ApiV2(base.Base): key: asset.json[key] for key in options if key in asset.json and key not in asset.related and key != 'id' } - fields['natural_key'] = asset.get_natural_key() for key in options: if key not in asset.related: @@ -122,8 +121,9 @@ class ApiV2(base.Base): # FIXME: use caching by url fields[key] = asset.related[key].get().get_natural_key() except exc.Forbidden: - log.warning("This object cannot be read: %s", asset.related[key]) - pass # FIXME: what if the fk is mandatory? + log.warning("This foreign key cannot be read: %s", asset.related[key]) + if options[key]['required']: + return None # This is a mandatory foreign key related = {} for key, related_endpoint in asset.related.items(): @@ -147,7 +147,7 @@ class ApiV2(base.Base): # FIXME: use caching by url data = rel.get(all_pages=True) except exc.Forbidden: - log.warning("This object cannot be read: %s", related_endpoint) + log.warning("These related objects cannot be read: %s", related_endpoint) continue if 'results' in data: @@ -162,6 +162,7 @@ class ApiV2(base.Base): if related: fields['related'] = related + fields['natural_key'] = asset.get_natural_key() return fields def _get_assets(self, resource, value): diff --git a/awxkit/awxkit/api/pages/page.py b/awxkit/awxkit/api/pages/page.py index dd73fc4d67..4a824ee290 100644 --- a/awxkit/awxkit/api/pages/page.py +++ b/awxkit/awxkit/api/pages/page.py @@ -331,7 +331,7 @@ class Page(object): # FIXME: use caching by url natural_key[key] = self.related[key].get().get_natural_key() except exc.Forbidden: - log.warning("This object cannot be read: %s", getattr(self, 'endpoint', '')) + log.warning("This foreign key cannot be read: %s", getattr(self, 'endpoint', '')) return None elif key in self: natural_key[key] = self[key] From 6958815f6e1d757972135dfe05f331c684265d46 Mon Sep 17 00:00:00 2001 From: Jeff Bradberry Date: Wed, 8 Apr 2020 17:48:07 -0400 Subject: [PATCH 51/73] Remove the $encrypted$ placeholders from export values --- awxkit/awxkit/api/pages/api.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/awxkit/awxkit/api/pages/api.py b/awxkit/awxkit/api/pages/api.py index f54010052d..d5314d0baf 100644 --- a/awxkit/awxkit/api/pages/api.py +++ b/awxkit/awxkit/api/pages/api.py @@ -71,6 +71,16 @@ def parse_description(desc): return options +def remove_encrypted(value): + if value == '$encrypted$': + return '' + if isinstance(value, list): + return [remove_encrypted(item) for item in value] + if isinstance(value, dict): + return {k: remove_encrypted(v) for k, v in value.items()} + return value + + class ApiV2(base.Base): # Common import/export methods @@ -163,7 +173,7 @@ class ApiV2(base.Base): fields['related'] = related fields['natural_key'] = asset.get_natural_key() - return fields + return remove_encrypted(fields) def _get_assets(self, resource, value): endpoint = getattr(self, resource) From 6387258da1b60cb38b12ee41e51b719dc96a6598 Mon Sep 17 00:00:00 2001 From: Jeff Bradberry Date: Wed, 8 Apr 2020 17:48:36 -0400 Subject: [PATCH 52/73] Only wait for Project updates not other kinds of objects that have a status. --- awxkit/awxkit/api/pages/api.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/awxkit/awxkit/api/pages/api.py b/awxkit/awxkit/api/pages/api.py index d5314d0baf..b4a786d7a6 100644 --- a/awxkit/awxkit/api/pages/api.py +++ b/awxkit/awxkit/api/pages/api.py @@ -239,10 +239,10 @@ class ApiV2(base.Base): if frozen_key is not None and frozen_key not in self._natural_key and fetch: pass # FIXME - from awxkit.api.mixins import has_status + from awxkit.api.pages import projects _page = self._natural_key.get(frozen_key) - if isinstance(_page, has_status.HasStatus) and not _page.is_completed: + if isinstance(_page, projects.Project) and not _page.is_completed: _page.wait_until_completed() _page = _page.get() self._natural_key[frozen_key] = _page From 471dc2babf555274f175c51fcbdabef2cec781f3 Mon Sep 17 00:00:00 2001 From: Jeff Bradberry Date: Thu, 9 Apr 2020 15:20:59 -0400 Subject: [PATCH 53/73] Deal with relations that we cannot resolve such as due to a lack of permissions. If there is a foreign key to something where we don't have sufficient read permissions, we now drop the parent object from the export. --- awxkit/awxkit/api/pages/api.py | 14 ++++++++++---- awxkit/awxkit/api/pages/page.py | 2 +- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/awxkit/awxkit/api/pages/api.py b/awxkit/awxkit/api/pages/api.py index b4a786d7a6..ddd01803bf 100644 --- a/awxkit/awxkit/api/pages/api.py +++ b/awxkit/awxkit/api/pages/api.py @@ -129,11 +129,13 @@ class ApiV2(base.Base): continue try: # FIXME: use caching by url - fields[key] = asset.related[key].get().get_natural_key() + natural_key = asset.related[key].get().get_natural_key() except exc.Forbidden: log.warning("This foreign key cannot be read: %s", asset.related[key]) - if options[key]['required']: - return None # This is a mandatory foreign key + return None + if natural_key is None: + return None # This is an unresolvable foreign key + fields[key] = natural_key related = {} for key, related_endpoint in asset.related.items(): @@ -172,7 +174,11 @@ class ApiV2(base.Base): if related: fields['related'] = related - fields['natural_key'] = asset.get_natural_key() + natural_key = asset.get_natural_key() + if natural_key is None: + return None + fields['natural_key'] = natural_key + return remove_encrypted(fields) def _get_assets(self, resource, value): diff --git a/awxkit/awxkit/api/pages/page.py b/awxkit/awxkit/api/pages/page.py index 4a824ee290..91098abe8d 100644 --- a/awxkit/awxkit/api/pages/page.py +++ b/awxkit/awxkit/api/pages/page.py @@ -331,7 +331,7 @@ class Page(object): # FIXME: use caching by url natural_key[key] = self.related[key].get().get_natural_key() except exc.Forbidden: - log.warning("This foreign key cannot be read: %s", getattr(self, 'endpoint', '')) + log.warning("This foreign key cannot be read: %s", self.related[key]) return None elif key in self: natural_key[key] = self[key] From 201de4e18a08c5b3c708eb86c6b7f78e266e1f2d Mon Sep 17 00:00:00 2001 From: Jeff Bradberry Date: Thu, 9 Apr 2020 15:22:41 -0400 Subject: [PATCH 54/73] Attempt to deal with foreign keys that get mislabeled as type integer such as through the use of DeprecatedCredentialField. --- awxkit/awxkit/api/pages/api.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/awxkit/awxkit/api/pages/api.py b/awxkit/awxkit/api/pages/api.py index ddd01803bf..da824e999d 100644 --- a/awxkit/awxkit/api/pages/api.py +++ b/awxkit/awxkit/api/pages/api.py @@ -266,20 +266,24 @@ class ApiV2(base.Base): for field, value in asset.items(): if field not in options: continue - if options[field]['type'] == 'id': + if options[field]['type'] in ('id', 'integer') and isinstance(value, dict): page = self._get_by_natural_key(value) post_data[field] = page['id'] if page is not None else None else: post_data[field] = value page = self._get_by_natural_key(asset['natural_key'], fetch=False) - if page is None: - if resource == 'users': - # We should only impose a default password if the resource doesn't exist. - post_data.setdefault('password', 'abc123') - page = endpoint.post(post_data) - else: - page = page.put(post_data) + try: + if page is None: + if resource == 'users': + # We should only impose a default password if the resource doesn't exist. + post_data.setdefault('password', 'abc123') + page = endpoint.post(post_data) + else: + page = page.put(post_data) + except exc.Common: + log.exception("post_data: %r", post_data) + raise self._register_page(page) From bb66e4633db7c20301abf3af83d29f083c4548cd Mon Sep 17 00:00:00 2001 From: Jeff Bradberry Date: Mon, 13 Apr 2020 15:31:11 -0400 Subject: [PATCH 55/73] Split _get_options into two pieces --- awxkit/awxkit/api/pages/api.py | 35 ++++++++++++++++++++-------------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/awxkit/awxkit/api/pages/api.py b/awxkit/awxkit/api/pages/api.py index da824e999d..30907a19d2 100644 --- a/awxkit/awxkit/api/pages/api.py +++ b/awxkit/awxkit/api/pages/api.py @@ -89,11 +89,7 @@ class ApiV2(base.Base): if getattr(self, '_options', None) is None: self._options = {} - if isinstance(_page, page.TentativePage): - url = str(_page) - else: - url = _page.url - + url = _page.url if isinstance(_page, page.Page) else str(_page) if url in self._options: return self._options[url] @@ -101,13 +97,21 @@ class ApiV2(base.Base): warning = options.r.headers.get('Warning', '') if '299' in warning and 'deprecated' in warning: return self._options.setdefault(url, None) - if 'POST' not in options.r.headers.get('Allow', ''): - return self._options.setdefault(url, None) - if 'POST' in options.json['actions']: - return self._options.setdefault(url, options.json['actions']['POST']) + return self._options.setdefault(url, options) + + def _get_post_fields(self, _page): + options_page = self._get_options(_page) + if options_page is None: + return None + + if 'POST' not in options_page.r.headers.get('Allow', ''): + return None + + if 'POST' in options_page.json['actions']: + return options_page.json['actions']['POST'] else: - return self._options.setdefault(url, parse_description(options.json['description'])) + return parse_description(options_page.json['description']) # Export methods @@ -143,7 +147,7 @@ class ApiV2(base.Base): continue rel = related_endpoint._create() - related_options = self._get_options(related_endpoint) + related_options = self._get_post_fields(related_endpoint) if related_options is None: # This is a read-only endpoint. continue is_attach = 'id' in related_options # This is not a create-only endpoint. @@ -183,7 +187,7 @@ class ApiV2(base.Base): def _get_assets(self, resource, value): endpoint = getattr(self, resource) - options = self._get_options(endpoint) + options = self._get_post_fields(endpoint) if options is None: return None @@ -232,7 +236,7 @@ class ApiV2(base.Base): def _register_existing_assets(self, resource): endpoint = getattr(self, resource) - options = self._get_options(endpoint) + options = self._get_post_fields(endpoint) if options is None: return @@ -255,11 +259,14 @@ class ApiV2(base.Base): return _page def _create_assets(self, data, resource): + # FIXME: this method should work with any list-create + # endpoint, so that we can use it with create relations, e.g. WFJT Nodes + if resource not in data or resource not in EXPORTABLE_RESOURCES: return endpoint = getattr(self, resource) - options = self._get_options(endpoint) + options = self._get_post_fields(endpoint) assets = data[resource] for asset in assets: post_data = {} From 1300d38e47b8353f1b91c4516a14f164aa9e451b Mon Sep 17 00:00:00 2001 From: Jeff Bradberry Date: Mon, 13 Apr 2020 15:40:47 -0400 Subject: [PATCH 56/73] Introduce a new PageCache object and split out get_post_fields into its own utility function. --- awxkit/awxkit/api/pages/api.py | 67 ++++----------------------------- awxkit/awxkit/api/pages/page.py | 21 +++++++++++ awxkit/awxkit/api/utils.py | 39 +++++++++++++++++++ 3 files changed, 68 insertions(+), 59 deletions(-) create mode 100644 awxkit/awxkit/api/utils.py diff --git a/awxkit/awxkit/api/pages/api.py b/awxkit/awxkit/api/pages/api.py index 30907a19d2..887420b8d7 100644 --- a/awxkit/awxkit/api/pages/api.py +++ b/awxkit/awxkit/api/pages/api.py @@ -1,15 +1,13 @@ import itertools import logging -import re from awxkit.api.resources import resources import awxkit.exceptions as exc from . import base from . import page +from .. import utils from ..mixins import has_create -descRE = re.compile(r'^[*] `(\w+)`: [^(]*\((\w+), ([^)]+)\)') - log = logging.getLogger(__name__) @@ -60,59 +58,8 @@ class Api(base.Base): page.register_page(resources.api, Api) -def parse_description(desc): - options = {} - for line in desc[desc.index('POST'):].splitlines(): - match = descRE.match(line) - if not match: - continue - options[match.group(1)] = {'type': match.group(2), - 'required': match.group(3) == 'required'} - return options - - -def remove_encrypted(value): - if value == '$encrypted$': - return '' - if isinstance(value, list): - return [remove_encrypted(item) for item in value] - if isinstance(value, dict): - return {k: remove_encrypted(v) for k, v in value.items()} - return value - - class ApiV2(base.Base): - # Common import/export methods - - def _get_options(self, _page): - if getattr(self, '_options', None) is None: - self._options = {} - - url = _page.url if isinstance(_page, page.Page) else str(_page) - if url in self._options: - return self._options[url] - - options = _page.options() - warning = options.r.headers.get('Warning', '') - if '299' in warning and 'deprecated' in warning: - return self._options.setdefault(url, None) - - return self._options.setdefault(url, options) - - def _get_post_fields(self, _page): - options_page = self._get_options(_page) - if options_page is None: - return None - - if 'POST' not in options_page.r.headers.get('Allow', ''): - return None - - if 'POST' in options_page.json['actions']: - return options_page.json['actions']['POST'] - else: - return parse_description(options_page.json['description']) - # Export methods def _serialize_asset(self, asset, options): @@ -147,7 +94,7 @@ class ApiV2(base.Base): continue rel = related_endpoint._create() - related_options = self._get_post_fields(related_endpoint) + related_options = utils.get_post_fields(related_endpoint, self._cache) if related_options is None: # This is a read-only endpoint. continue is_attach = 'id' in related_options # This is not a create-only endpoint. @@ -183,11 +130,11 @@ class ApiV2(base.Base): return None fields['natural_key'] = natural_key - return remove_encrypted(fields) + return utils.remove_encrypted(fields) def _get_assets(self, resource, value): endpoint = getattr(self, resource) - options = self._get_post_fields(endpoint) + options = utils.get_post_fields(endpoint, self._cache) if options is None: return None @@ -203,6 +150,8 @@ class ApiV2(base.Base): return [asset for asset in assets if asset is not None] def export_assets(self, **kwargs): + self._cache = page.PageCache() + # If no resource kwargs are explicitly used, export everything. all_resources = all(kwargs.get(resource) is None for resource in EXPORTABLE_RESOURCES) @@ -236,7 +185,7 @@ class ApiV2(base.Base): def _register_existing_assets(self, resource): endpoint = getattr(self, resource) - options = self._get_post_fields(endpoint) + options = utils.get_post_fields(endpoint, self._cache) if options is None: return @@ -266,7 +215,7 @@ class ApiV2(base.Base): return endpoint = getattr(self, resource) - options = self._get_post_fields(endpoint) + options = utils.get_post_fields(endpoint, self._cache) assets = data[resource] for asset in assets: post_data = {} diff --git a/awxkit/awxkit/api/pages/page.py b/awxkit/awxkit/api/pages/page.py index 91098abe8d..94943ec6b6 100644 --- a/awxkit/awxkit/api/pages/page.py +++ b/awxkit/awxkit/api/pages/page.py @@ -531,3 +531,24 @@ class TentativePage(str): def __ne__(self, other): return self.endpoint != other + + +class PageCache(object): + def __init__(self): + self.options = {} + + def get_options(self, page): + url = page.url if isinstance(page, Page) else str(page) + if url in self.options: + return self.options[url] + + try: + options = page.options() + except exc.Common: + return self.options.setdefault(url, None) + + warning = options.r.headers.get('Warning', '') + if '299' in warning and 'deprecated' in warning: + return self.options.setdefault(url, None) + + return self.options.setdefault(url, options) diff --git a/awxkit/awxkit/api/utils.py b/awxkit/awxkit/api/utils.py new file mode 100644 index 0000000000..1e295c1eb3 --- /dev/null +++ b/awxkit/awxkit/api/utils.py @@ -0,0 +1,39 @@ +import re + + +descRE = re.compile(r'^[*] `(\w+)`: [^(]*\((\w+), ([^)]+)\)') + + +def parse_description(desc): + options = {} + for line in desc[desc.index('POST'):].splitlines(): + match = descRE.match(line) + if not match: + continue + options[match.group(1)] = {'type': match.group(2), + 'required': match.group(3) == 'required'} + return options + + +def remove_encrypted(value): + if value == '$encrypted$': + return '' + if isinstance(value, list): + return [remove_encrypted(item) for item in value] + if isinstance(value, dict): + return {k: remove_encrypted(v) for k, v in value.items()} + return value + + +def get_post_fields(page, cache): + options_page = cache.get_options(page) + if options_page is None: + return None + + if 'POST' not in options_page.r.headers.get('Allow', ''): + return None + + if 'POST' in options_page.json['actions']: + return options_page.json['actions']['POST'] + else: + return parse_description(options_page.json['description']) From 14b5f63bd824785a9afeac4588057ae27b51e179 Mon Sep 17 00:00:00 2001 From: Jeff Bradberry Date: Mon, 13 Apr 2020 17:27:29 -0400 Subject: [PATCH 57/73] Use the new PageCache to store and reuse Page.get results --- awxkit/awxkit/api/pages/api.py | 33 +++++++++++------------ awxkit/awxkit/api/pages/page.py | 45 ++++++++++++++++++++++++++------ awxkit/awxkit/api/pages/roles.py | 14 +++++----- 3 files changed, 60 insertions(+), 32 deletions(-) diff --git a/awxkit/awxkit/api/pages/api.py b/awxkit/awxkit/api/pages/api.py index 887420b8d7..c7e90c4f28 100644 --- a/awxkit/awxkit/api/pages/api.py +++ b/awxkit/awxkit/api/pages/api.py @@ -78,14 +78,13 @@ class ApiV2(base.Base): for key in options: if key not in asset.related: continue - try: - # FIXME: use caching by url - natural_key = asset.related[key].get().get_natural_key() - except exc.Forbidden: - log.warning("This foreign key cannot be read: %s", asset.related[key]) - return None + + related_endpoint = self._cache.get_page(asset.related[key]) + if related_endpoint is None: + return None # This foreign key is unreadable + natural_key = related_endpoint.get_natural_key(self._cache) if natural_key is None: - return None # This is an unresolvable foreign key + return None # This foreign key has unresolvable dependencies fields[key] = natural_key related = {} @@ -106,16 +105,13 @@ class ApiV2(base.Base): else: continue - try: - # FIXME: use caching by url - data = rel.get(all_pages=True) - except exc.Forbidden: - log.warning("These related objects cannot be read: %s", related_endpoint) + data = self._cache.get_page(related_endpoint) + if data is None: continue if 'results' in data: results = ( - x.get_natural_key() if by_natural_key else self._serialize_asset(x, related_options) + x.get_natural_key(self._cache) if by_natural_key else self._serialize_asset(x, related_options) for x in data.results ) related[key] = [x for x in results if x is not None] @@ -125,7 +121,7 @@ class ApiV2(base.Base): if related: fields['related'] = related - natural_key = asset.get_natural_key() + natural_key = asset.get_natural_key(self._cache) if natural_key is None: return None fields['natural_key'] = natural_key @@ -144,7 +140,7 @@ class ApiV2(base.Base): pk = pk_or_name(self, resource, value) # TODO: decide whether to support multiple results = endpoint.get(id=pk).results else: - results = endpoint.get(all_pages=True).results + results = self._cache.get_page(endpoint).results assets = (self._serialize_asset(asset, options) for asset in results) return [asset for asset in assets if asset is not None] @@ -174,7 +170,7 @@ class ApiV2(base.Base): yield page_resource[page_cls] def _register_page(self, page): - natural_key = freeze(page.get_natural_key()) + natural_key = freeze(page.get_natural_key(self._cache)) # FIXME: we need to keep a reference for the case where we # don't have a natural key, so we can delete if natural_key is not None: @@ -189,7 +185,7 @@ class ApiV2(base.Base): if options is None: return - results = endpoint.get(all_pages=True).results + results = self._cache.get_page(endpoint).results for pg in results: self._register_page(pg) @@ -237,6 +233,7 @@ class ApiV2(base.Base): page = endpoint.post(post_data) else: page = page.put(post_data) + # FIXME: created pages need to be put in the cache except exc.Common: log.exception("post_data: %r", post_data) raise @@ -274,7 +271,7 @@ class ApiV2(base.Base): for item in related_set: data = {key: value for key, value in item.items() if key not in ('natural_key', 'related')} - endpoint.post(data) + endpoint.post(data) # FIXME: add the page to the cache # FIXME: deal with objects that themselves have relateds, e.g. WFJT Nodes # FIXME: deal with pruning existing relations that do not match the import set diff --git a/awxkit/awxkit/api/pages/page.py b/awxkit/awxkit/api/pages/page.py index 94943ec6b6..b4f0eaf0b9 100644 --- a/awxkit/awxkit/api/pages/page.py +++ b/awxkit/awxkit/api/pages/page.py @@ -317,9 +317,12 @@ class Page(object): page_cls = get_registered_page(endpoint) return page_cls(self.connection, endpoint=endpoint).get(**kw) - def get_natural_key(self): + def get_natural_key(self, cache=None): warn = "This object does not have a natural key: %s" + if cache is None: + cache = PageCache() + if not getattr(self, 'NATURAL_KEY', None): log.warning(warn, getattr(self, 'endpoint', '')) return None @@ -327,12 +330,10 @@ class Page(object): natural_key = {} for key in self.NATURAL_KEY: if key in self.related: - try: - # FIXME: use caching by url - natural_key[key] = self.related[key].get().get_natural_key() - except exc.Forbidden: - log.warning("This foreign key cannot be read: %s", self.related[key]) + related_endpoint = cache.get_page(self.related[key]) + if related_endpoint is None: return None + natural_key[key] = related_endpoint.get_natural_key(cache=cache) elif key in self: natural_key[key] = self[key] if not natural_key: @@ -401,7 +402,7 @@ class PageList(object): def create(self, *a, **kw): return self.__item_class__(self.connection).create(*a, **kw) - def get_natural_key(self): + def get_natural_key(self, cache=None): log.warning("This object does not have a natural key: %s", getattr(self, 'endpoint', '')) return None @@ -536,19 +537,47 @@ class TentativePage(str): class PageCache(object): def __init__(self): self.options = {} + self.pages_by_url = {} def get_options(self, page): - url = page.url if isinstance(page, Page) else str(page) + url = page.endpoint if isinstance(page, Page) else str(page) if url in self.options: return self.options[url] try: options = page.options() except exc.Common: + log.error("This endpoint raised an error: %s", url) return self.options.setdefault(url, None) warning = options.r.headers.get('Warning', '') if '299' in warning and 'deprecated' in warning: + log.warning("This endpoint is deprecated: %s", url) return self.options.setdefault(url, None) return self.options.setdefault(url, options) + + def set_page(self, page): + self.pages_by_url[page.endpoint] = page + if 'results' in page: + for p in page.results: + self.set_page(p) + return page + + def get_page(self, page): + url = page.endpoint if isinstance(page, Page) else str(page) + if url in self.pages_by_url: + return self.pages_by_url[url] + + try: + page = page.get(all_pages=True) + except exc.Common: + log.error("This endpoint raised an error: %s", url) + return self.pages_by_url.setdefault(url, None) + + warning = page.r.headers.get('Warning', '') + if '299' in warning and 'deprecated' in warning: + log.warning("This endpoint is deprecated: %s", url) + return self.pages_by_url.setdefault(url, None) + + return self.set_page(page) diff --git a/awxkit/awxkit/api/pages/roles.py b/awxkit/awxkit/api/pages/roles.py index d93de4cad1..530dfc36e9 100644 --- a/awxkit/awxkit/api/pages/roles.py +++ b/awxkit/awxkit/api/pages/roles.py @@ -9,18 +9,20 @@ class Role(base.Base): NATURAL_KEY = ('name',) - def get_natural_key(self): - natural_key = super(Role, self).get_natural_key() + def get_natural_key(self, cache=None): + if cache is None: + cache = page.PageCache() + + natural_key = super(Role, self).get_natural_key(cache=cache) related_objs = [ related for name, related in self.related.items() if name not in ('users', 'teams') ] if related_objs: - try: - # FIXME: use caching by url - natural_key['content_object'] = related_objs[0].get().get_natural_key() - except exc.Forbidden: + related_endpoint = cache.get_page(related_objs[0]) + if related_endpoint is None: return None + natural_key['content_object'] = related_endpoint.get_natural_key(cache=cache) return natural_key From f7825aefebaa6bebc965d3dfdb18ca1d677a542f Mon Sep 17 00:00:00 2001 From: Jeff Bradberry Date: Mon, 13 Apr 2020 17:57:07 -0400 Subject: [PATCH 58/73] Avoid doing an OPTIONS call unless we know it is a related type we export --- awxkit/awxkit/api/pages/api.py | 9 +++++++-- awxkit/awxkit/api/pages/roles.py | 1 - 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/awxkit/awxkit/api/pages/api.py b/awxkit/awxkit/api/pages/api.py index c7e90c4f28..89751b734d 100644 --- a/awxkit/awxkit/api/pages/api.py +++ b/awxkit/awxkit/api/pages/api.py @@ -93,14 +93,19 @@ class ApiV2(base.Base): continue rel = related_endpoint._create() + is_relation = rel.__class__.__name__ in EXPORTABLE_RELATIONS + is_dependent = rel.__class__.__name__ in EXPORTABLE_DEPENDENT_OBJECTS + if not (is_relation or is_dependent): + continue + related_options = utils.get_post_fields(related_endpoint, self._cache) if related_options is None: # This is a read-only endpoint. continue is_attach = 'id' in related_options # This is not a create-only endpoint. - if rel.__class__.__name__ in EXPORTABLE_RELATIONS and is_attach: + if is_relation and is_attach: by_natural_key = True - elif rel.__class__.__name__ in EXPORTABLE_DEPENDENT_OBJECTS: + elif is_dependent: by_natural_key = False else: continue diff --git a/awxkit/awxkit/api/pages/roles.py b/awxkit/awxkit/api/pages/roles.py index 530dfc36e9..104af818b1 100644 --- a/awxkit/awxkit/api/pages/roles.py +++ b/awxkit/awxkit/api/pages/roles.py @@ -1,5 +1,4 @@ from awxkit.api.resources import resources -import awxkit.exceptions as exc from . import base from . import page From 9fc1a4bb44a6f52aecbeac6d9234f8be3d47d3a4 Mon Sep 17 00:00:00 2001 From: Jeff Bradberry Date: Wed, 15 Apr 2020 14:29:29 -0400 Subject: [PATCH 59/73] Change the method that exports list views to take a PageList or TentativePage This will allow the related m2m views to also use this method, with a bit of effort. Also, remove the use of pk_or_name in favor of a new method that reduces the number of api calls. --- awxkit/awxkit/api/pages/api.py | 33 ++++++++++++++++++++------------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/awxkit/awxkit/api/pages/api.py b/awxkit/awxkit/api/pages/api.py index 89751b734d..fd8362fb16 100644 --- a/awxkit/awxkit/api/pages/api.py +++ b/awxkit/awxkit/api/pages/api.py @@ -133,23 +133,27 @@ class ApiV2(base.Base): return utils.remove_encrypted(fields) - def _get_assets(self, resource, value): - endpoint = getattr(self, resource) - options = utils.get_post_fields(endpoint, self._cache) - if options is None: + def _export_list(self, endpoint): + post_fields = utils.get_post_fields(endpoint, self._cache) + if post_fields is None: return None - if value: - from awxkit.cli.options import pk_or_name + if isinstance(endpoint, page.TentativePage): + endpoint = self._cache.get_page(endpoint) + if endpoint is None: + return None - pk = pk_or_name(self, resource, value) # TODO: decide whether to support multiple - results = endpoint.get(id=pk).results - else: - results = self._cache.get_page(endpoint).results - - assets = (self._serialize_asset(asset, options) for asset in results) + assets = (self._serialize_asset(asset, post_fields) for asset in endpoint.results) return [asset for asset in assets if asset is not None] + def _filtered_list(self, endpoint, value): + if isinstance(value, int) or value.isdecimal(): + return endpoint.get(id=int(value)) + options = self._cache.get_options(endpoint) + identifier = next(field for field in options['search_fields'] + if field in ('name', 'username', 'hostname')) + return endpoint.get(**{identifier: value}) + def export_assets(self, **kwargs): self._cache = page.PageCache() @@ -160,7 +164,10 @@ class ApiV2(base.Base): for resource in EXPORTABLE_RESOURCES: value = kwargs.get(resource) if all_resources or value is not None: - data[resource] = self._get_assets(resource, value) + endpoint = getattr(self, resource) + if value: + endpoint = self._filtered_list(endpoint, value) + data[resource] = self._export_list(endpoint) return data From 301f15bfdd92e4f2fc6120ae18e74a876270ea91 Mon Sep 17 00:00:00 2001 From: Jeff Bradberry Date: Wed, 22 Apr 2020 14:32:58 -0400 Subject: [PATCH 60/73] Rename some things for consistency --- awxkit/awxkit/api/pages/api.py | 84 +++++++++++++++++----------------- 1 file changed, 42 insertions(+), 42 deletions(-) diff --git a/awxkit/awxkit/api/pages/api.py b/awxkit/awxkit/api/pages/api.py index fd8362fb16..081f5b15e3 100644 --- a/awxkit/awxkit/api/pages/api.py +++ b/awxkit/awxkit/api/pages/api.py @@ -62,46 +62,46 @@ class ApiV2(base.Base): # Export methods - def _serialize_asset(self, asset, options): + def _export(self, _page, post_fields): # Drop any (credential_type) assets that are being managed by the Tower instance. - if asset.json.get('managed_by_tower'): + if _page.json.get('managed_by_tower'): return None - if options is None: # Deprecated endpoint or insufficient permissions + if post_fields is None: # Deprecated endpoint or insufficient permissions return None - # Note: doing asset[key] automatically parses json blob strings, which can be a problem. + # Note: doing _page[key] automatically parses json blob strings, which can be a problem. fields = { - key: asset.json[key] for key in options - if key in asset.json and key not in asset.related and key != 'id' + key: _page.json[key] for key in post_fields + if key in _page.json and key not in _page.related and key != 'id' } - for key in options: - if key not in asset.related: + for key in post_fields: + if key not in _page.related: continue - related_endpoint = self._cache.get_page(asset.related[key]) - if related_endpoint is None: + rel_endpoint = self._cache.get_page(_page.related[key]) + if rel_endpoint is None: return None # This foreign key is unreadable - natural_key = related_endpoint.get_natural_key(self._cache) + natural_key = rel_endpoint.get_natural_key(self._cache) if natural_key is None: return None # This foreign key has unresolvable dependencies fields[key] = natural_key related = {} - for key, related_endpoint in asset.related.items(): - if key in options or not related_endpoint: + for key, rel_endpoint in _page.related.items(): + if key in post_fields or not rel_endpoint: continue - rel = related_endpoint._create() + rel = rel_endpoint._create() is_relation = rel.__class__.__name__ in EXPORTABLE_RELATIONS is_dependent = rel.__class__.__name__ in EXPORTABLE_DEPENDENT_OBJECTS if not (is_relation or is_dependent): continue - related_options = utils.get_post_fields(related_endpoint, self._cache) - if related_options is None: # This is a read-only endpoint. + rel_post_fields = utils.get_post_fields(rel_endpoint, self._cache) + if rel_post_fields is None: # This is a read-only endpoint. continue - is_attach = 'id' in related_options # This is not a create-only endpoint. + is_attach = 'id' in rel_post_fields # This is not a create-only endpoint. if is_relation and is_attach: by_natural_key = True @@ -110,23 +110,23 @@ class ApiV2(base.Base): else: continue - data = self._cache.get_page(related_endpoint) - if data is None: + rel_page = self._cache.get_page(rel_endpoint) + if rel_page is None: continue - if 'results' in data: + if 'results' in rel_page: results = ( - x.get_natural_key(self._cache) if by_natural_key else self._serialize_asset(x, related_options) - for x in data.results + x.get_natural_key(self._cache) if by_natural_key else self._export(x, rel_post_fields) + for x in rel_page.results ) related[key] = [x for x in results if x is not None] else: - related[key] = data.json + related[key] = rel_page.json if related: fields['related'] = related - natural_key = asset.get_natural_key(self._cache) + natural_key = _page.get_natural_key(self._cache) if natural_key is None: return None fields['natural_key'] = natural_key @@ -143,7 +143,7 @@ class ApiV2(base.Base): if endpoint is None: return None - assets = (self._serialize_asset(asset, post_fields) for asset in endpoint.results) + assets = (self._export(asset, post_fields) for asset in endpoint.results) return [asset for asset in assets if asset is not None] def _filtered_list(self, endpoint, value): @@ -206,13 +206,7 @@ class ApiV2(base.Base): if frozen_key is not None and frozen_key not in self._natural_key and fetch: pass # FIXME - from awxkit.api.pages import projects - _page = self._natural_key.get(frozen_key) - if isinstance(_page, projects.Project) and not _page.is_completed: - _page.wait_until_completed() - _page = _page.get() - self._natural_key[frozen_key] = _page return _page def _create_assets(self, data, resource): @@ -231,26 +225,27 @@ class ApiV2(base.Base): if field not in options: continue if options[field]['type'] in ('id', 'integer') and isinstance(value, dict): - page = self._get_by_natural_key(value) - post_data[field] = page['id'] if page is not None else None + _page = self._get_by_natural_key(value) + post_data[field] = _page['id'] if _page is not None else None else: post_data[field] = value - page = self._get_by_natural_key(asset['natural_key'], fetch=False) + _page = self._get_by_natural_key(asset['natural_key'], fetch=False) try: - if page is None: + if _page is None: if resource == 'users': # We should only impose a default password if the resource doesn't exist. post_data.setdefault('password', 'abc123') - page = endpoint.post(post_data) + _page = endpoint.post(post_data) else: - page = page.put(post_data) + _page = _page.put(post_data) # FIXME: created pages need to be put in the cache - except exc.Common: - log.exception("post_data: %r", post_data) - raise + except exc.Common as e: + log.error("Object import failed: %s.", e) + log.debug("post_data: %r", post_data) + continue - self._register_page(page) + self._register_page(_page) def _assign_roles(self, page, roles): role_endpoint = page.json['related']['roles'] @@ -278,7 +273,10 @@ class ApiV2(base.Base): rel_page = self._get_by_natural_key(item) if rel_page is None: continue # FIXME - endpoint.post({'id': rel_page['id']}) + try: + endpoint.post({'id': rel_page['id']}) + except exc.NoContent: # desired exception on successful (dis)association + pass else: # It is a create set for item in related_set: data = {key: value for key, value in item.items() @@ -301,6 +299,8 @@ class ApiV2(base.Base): self._assign_related(page, name, S) def import_assets(self, data): + self._cache = page.PageCache() + for resource in self._dependent_resources(data): self._register_existing_assets(resource) self._create_assets(data, resource) From 66bc947adbc5090df057e51e76f8c1b12e5c8c14 Mon Sep 17 00:00:00 2001 From: Jeff Bradberry Date: Wed, 22 Apr 2020 16:46:03 -0400 Subject: [PATCH 61/73] Reuse _import_list for import of related full objects e.g. WFJT Nodes. Also rename _import_list from _create_assets. --- awxkit/awxkit/api/pages/api.py | 70 +++++++++++++++++----------------- 1 file changed, 34 insertions(+), 36 deletions(-) diff --git a/awxkit/awxkit/api/pages/api.py b/awxkit/awxkit/api/pages/api.py index 081f5b15e3..72d1fa4bf5 100644 --- a/awxkit/awxkit/api/pages/api.py +++ b/awxkit/awxkit/api/pages/api.py @@ -191,15 +191,14 @@ class ApiV2(base.Base): self._natural_key[natural_key] = page - def _register_existing_assets(self, resource): - endpoint = getattr(self, resource) - options = utils.get_post_fields(endpoint, self._cache) - if options is None: + def _register_existing_assets(self, endpoint): + post_fields = utils.get_post_fields(endpoint, self._cache) + if post_fields is None: return results = self._cache.get_page(endpoint).results - for pg in results: - self._register_page(pg) + for _page in results: + self._register_page(_page) def _get_by_natural_key(self, key, fetch=True): frozen_key = freeze(key) @@ -209,22 +208,14 @@ class ApiV2(base.Base): _page = self._natural_key.get(frozen_key) return _page - def _create_assets(self, data, resource): - # FIXME: this method should work with any list-create - # endpoint, so that we can use it with create relations, e.g. WFJT Nodes - - if resource not in data or resource not in EXPORTABLE_RESOURCES: - return - - endpoint = getattr(self, resource) - options = utils.get_post_fields(endpoint, self._cache) - assets = data[resource] + def _import_list(self, endpoint, assets): + post_fields = utils.get_post_fields(endpoint, self._cache) for asset in assets: post_data = {} for field, value in asset.items(): - if field not in options: + if field not in post_fields: continue - if options[field]['type'] in ('id', 'integer') and isinstance(value, dict): + if post_fields[field]['type'] in ('id', 'integer') and isinstance(value, dict): _page = self._get_by_natural_key(value) post_data[field] = _page['id'] if _page is not None else None else: @@ -233,7 +224,7 @@ class ApiV2(base.Base): _page = self._get_by_natural_key(asset['natural_key'], fetch=False) try: if _page is None: - if resource == 'users': + if asset['natural_key']['type'] == 'user': # We should only impose a default password if the resource doesn't exist. post_data.setdefault('password', 'abc123') _page = endpoint.post(post_data) @@ -262,52 +253,59 @@ class ApiV2(base.Base): else: pass # admin role - def _assign_related(self, page, name, related_set): - endpoint = page.related[name] + def _assign_related(self, _page, name, related_set): + endpoint = _page.related[name] if isinstance(related_set, dict): # Relateds that are just json blobs, e.g. survey_spec endpoint.post(related_set) return if 'natural_key' not in related_set[0]: # It is an attach set + # Try to impedance match + related = endpoint.get(all_pages=True) + existing = {rel['id'] for rel in related.results} for item in related_set: rel_page = self._get_by_natural_key(item) if rel_page is None: continue # FIXME + if rel_page['id'] in existing: + continue try: - endpoint.post({'id': rel_page['id']}) + post_data = {'id': rel_page['id']} + endpoint.post(post_data) except exc.NoContent: # desired exception on successful (dis)association pass + except exc.Common as e: + log.error("Object association failed: %s.", e) + log.debug("post_data: %r", post_data) + raise else: # It is a create set - for item in related_set: - data = {key: value for key, value in item.items() - if key not in ('natural_key', 'related')} - endpoint.post(data) # FIXME: add the page to the cache - # FIXME: deal with objects that themselves have relateds, e.g. WFJT Nodes + self._import_list(endpoint, related_set) # FIXME: deal with pruning existing relations that do not match the import set - def _assign_related_assets(self, resource, assets): + def _assign_related_assets(self, assets): for asset in assets: - page = self._get_by_natural_key(asset['natural_key']) - # FIXME: deal with `page is None` case + _page = self._get_by_natural_key(asset['natural_key']) + # FIXME: deal with `_page is None` case for name, S in asset.get('related', {}).items(): if not S: continue if name == 'roles': - self._assign_roles(page, S) + self._assign_roles(_page, S) else: - self._assign_related(page, name, S) + self._assign_related(_page, name, S) def import_assets(self, data): self._cache = page.PageCache() for resource in self._dependent_resources(data): - self._register_existing_assets(resource) - self._create_assets(data, resource) + endpoint = getattr(self, resource) + self._register_existing_assets(endpoint) + self._import_list(endpoint, data.get(resource) or []) # FIXME: should we delete existing unpatched assets? - for resource, assets in data.items(): - self._assign_related_assets(resource, assets) + for assets in data.values(): + self._assign_related_assets(assets) page.register_page(resources.v2, ApiV2) From 57aff6394c93f6dd8c7b242ca8ec0024f53f2635 Mon Sep 17 00:00:00 2001 From: Jeff Bradberry Date: Thu, 23 Apr 2020 11:11:35 -0400 Subject: [PATCH 62/73] Log an error and continue when a related object is not found --- awxkit/awxkit/api/pages/api.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/awxkit/awxkit/api/pages/api.py b/awxkit/awxkit/api/pages/api.py index 72d1fa4bf5..4c70ed144d 100644 --- a/awxkit/awxkit/api/pages/api.py +++ b/awxkit/awxkit/api/pages/api.py @@ -286,7 +286,9 @@ class ApiV2(base.Base): def _assign_related_assets(self, assets): for asset in assets: _page = self._get_by_natural_key(asset['natural_key']) - # FIXME: deal with `_page is None` case + if _page is None: + log.error("Related object with natural key not found: %r", asset['natural_key']) + continue for name, S in asset.get('related', {}).items(): if not S: continue From 86afa5cf424646cf9d0ab7ed445baecf199b7c9e Mon Sep 17 00:00:00 2001 From: Jeff Bradberry Date: Thu, 23 Apr 2020 13:23:49 -0400 Subject: [PATCH 63/73] Make more use of the PageCache for imports --- awxkit/awxkit/api/pages/api.py | 49 ++++++--------------------------- awxkit/awxkit/api/pages/page.py | 13 +++++++++ awxkit/awxkit/api/utils.py | 6 ++++ 3 files changed, 27 insertions(+), 41 deletions(-) diff --git a/awxkit/awxkit/api/pages/api.py b/awxkit/awxkit/api/pages/api.py index 4c70ed144d..75dbe26bce 100644 --- a/awxkit/awxkit/api/pages/api.py +++ b/awxkit/awxkit/api/pages/api.py @@ -44,12 +44,6 @@ EXPORTABLE_DEPENDENT_OBJECTS = [ ] -def freeze(key): - if key is None: - return None - return frozenset((k, freeze(v) if isinstance(v, dict) else v) for k, v in key.items()) - - class Api(base.Base): pass @@ -181,33 +175,6 @@ class ApiV2(base.Base): for page_cls in itertools.chain(*has_create.page_creation_order(*data_pages)): yield page_resource[page_cls] - def _register_page(self, page): - natural_key = freeze(page.get_natural_key(self._cache)) - # FIXME: we need to keep a reference for the case where we - # don't have a natural key, so we can delete - if natural_key is not None: - if getattr(self, '_natural_key', None) is None: - self._natural_key = {} - - self._natural_key[natural_key] = page - - def _register_existing_assets(self, endpoint): - post_fields = utils.get_post_fields(endpoint, self._cache) - if post_fields is None: - return - - results = self._cache.get_page(endpoint).results - for _page in results: - self._register_page(_page) - - def _get_by_natural_key(self, key, fetch=True): - frozen_key = freeze(key) - if frozen_key is not None and frozen_key not in self._natural_key and fetch: - pass # FIXME - - _page = self._natural_key.get(frozen_key) - return _page - def _import_list(self, endpoint, assets): post_fields = utils.get_post_fields(endpoint, self._cache) for asset in assets: @@ -216,12 +183,12 @@ class ApiV2(base.Base): if field not in post_fields: continue if post_fields[field]['type'] in ('id', 'integer') and isinstance(value, dict): - _page = self._get_by_natural_key(value) + _page = self._cache.get_by_natural_key(value) post_data[field] = _page['id'] if _page is not None else None else: post_data[field] = value - _page = self._get_by_natural_key(asset['natural_key'], fetch=False) + _page = self._cache.get_by_natural_key(asset['natural_key']) try: if _page is None: if asset['natural_key']['type'] == 'user': @@ -230,20 +197,19 @@ class ApiV2(base.Base): _page = endpoint.post(post_data) else: _page = _page.put(post_data) - # FIXME: created pages need to be put in the cache except exc.Common as e: log.error("Object import failed: %s.", e) log.debug("post_data: %r", post_data) continue - self._register_page(_page) + self._cache.set_page(_page) def _assign_roles(self, page, roles): role_endpoint = page.json['related']['roles'] for role in roles: if 'content_object' not in role: continue # admin role - obj_page = self._get_by_natural_key(role['content_object']) + obj_page = self._cache.get_by_natural_key(role['content_object']) if obj_page is not None: role_page = obj_page.get_object_role(role['name'], by_name=True) try: @@ -264,7 +230,7 @@ class ApiV2(base.Base): related = endpoint.get(all_pages=True) existing = {rel['id'] for rel in related.results} for item in related_set: - rel_page = self._get_by_natural_key(item) + rel_page = self._cache.get_by_natural_key(item) if rel_page is None: continue # FIXME if rel_page['id'] in existing: @@ -285,7 +251,7 @@ class ApiV2(base.Base): def _assign_related_assets(self, assets): for asset in assets: - _page = self._get_by_natural_key(asset['natural_key']) + _page = self._cache.get_by_natural_key(asset['natural_key']) if _page is None: log.error("Related object with natural key not found: %r", asset['natural_key']) continue @@ -302,7 +268,8 @@ class ApiV2(base.Base): for resource in self._dependent_resources(data): endpoint = getattr(self, resource) - self._register_existing_assets(endpoint) + # Load up existing objects, so that we can try to update or link to them + self._cache.get_page(endpoint) self._import_list(endpoint, data.get(resource) or []) # FIXME: should we delete existing unpatched assets? diff --git a/awxkit/awxkit/api/pages/page.py b/awxkit/awxkit/api/pages/page.py index b4f0eaf0b9..814297a4eb 100644 --- a/awxkit/awxkit/api/pages/page.py +++ b/awxkit/awxkit/api/pages/page.py @@ -15,6 +15,7 @@ from awxkit.utils import ( is_list_or_tuple, to_str ) +from awxkit.api import utils from awxkit.api.client import Connection from awxkit.api.registry import URLRegistry from awxkit.config import config @@ -360,6 +361,8 @@ def exception_from_status_code(status_code): class PageList(object): + NATURAL_KEY = None + @property def __item_class__(self): """Returns the class representing a single 'Page' item @@ -538,6 +541,7 @@ class PageCache(object): def __init__(self): self.options = {} self.pages_by_url = {} + self.pages_by_natural_key = {} def get_options(self, page): url = page.endpoint if isinstance(page, Page) else str(page) @@ -559,6 +563,10 @@ class PageCache(object): def set_page(self, page): self.pages_by_url[page.endpoint] = page + if getattr(page, 'NATURAL_KEY', None): + natural_key = page.get_natural_key(cache=self) + if natural_key is not None: + self.pages_by_natural_key[utils.freeze(natural_key)] = page.endpoint if 'results' in page: for p in page.results: self.set_page(p) @@ -581,3 +589,8 @@ class PageCache(object): return self.pages_by_url.setdefault(url, None) return self.set_page(page) + + def get_by_natural_key(self, natural_key): + endpoint = self.pages_by_natural_key.get(utils.freeze(natural_key)) + if endpoint: + return self.get_page(endpoint) diff --git a/awxkit/awxkit/api/utils.py b/awxkit/awxkit/api/utils.py index 1e295c1eb3..355cdb0158 100644 --- a/awxkit/awxkit/api/utils.py +++ b/awxkit/awxkit/api/utils.py @@ -4,6 +4,12 @@ import re descRE = re.compile(r'^[*] `(\w+)`: [^(]*\((\w+), ([^)]+)\)') +def freeze(key): + if key is None: + return None + return frozenset((k, freeze(v) if isinstance(v, dict) else v) for k, v in key.items()) + + def parse_description(desc): options = {} for line in desc[desc.index('POST'):].splitlines(): From a531b85b31f71df1538e32a20ec8236eca3cb67d Mon Sep 17 00:00:00 2001 From: Jeff Bradberry Date: Thu, 23 Apr 2020 15:32:14 -0400 Subject: [PATCH 64/73] Redo waiting until Project updates are complete --- awxkit/awxkit/api/pages/api.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/awxkit/awxkit/api/pages/api.py b/awxkit/awxkit/api/pages/api.py index 75dbe26bce..218bc81658 100644 --- a/awxkit/awxkit/api/pages/api.py +++ b/awxkit/awxkit/api/pages/api.py @@ -195,9 +195,14 @@ class ApiV2(base.Base): # We should only impose a default password if the resource doesn't exist. post_data.setdefault('password', 'abc123') _page = endpoint.post(post_data) + if asset['natural_key']['type'] == 'project': + # When creating a project, we need to wait for its + # first project update to finish so that associated + # JTs have valid options for playbook names + _page.wait_until_completed() else: _page = _page.put(post_data) - except exc.Common as e: + except (exc.Common, AssertionError) as e: log.error("Object import failed: %s.", e) log.debug("post_data: %r", post_data) continue From 76fb605dcdb33438942a4acc67e982bf96af723b Mon Sep 17 00:00:00 2001 From: Jeff Bradberry Date: Tue, 28 Apr 2020 13:50:12 -0400 Subject: [PATCH 65/73] Deal with exports involving foreign keys where you don't have permission --- awxkit/awxkit/api/pages/api.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/awxkit/awxkit/api/pages/api.py b/awxkit/awxkit/api/pages/api.py index 218bc81658..28e9393462 100644 --- a/awxkit/awxkit/api/pages/api.py +++ b/awxkit/awxkit/api/pages/api.py @@ -61,6 +61,7 @@ class ApiV2(base.Base): if _page.json.get('managed_by_tower'): return None if post_fields is None: # Deprecated endpoint or insufficient permissions + log.error("Object export failed: %s", _page.endpoint) return None # Note: doing _page[key] automatically parses json blob strings, which can be a problem. @@ -74,8 +75,12 @@ class ApiV2(base.Base): continue rel_endpoint = self._cache.get_page(_page.related[key]) - if rel_endpoint is None: - return None # This foreign key is unreadable + if rel_endpoint is None: # This foreign key is unreadable + if post_fields[key].get('required'): + log.error("Foreign key export failed: %s", _page.related[key]) + return None + log.error("Foreign key export failed, setting to null: %s", _page.related[key]) + continue natural_key = rel_endpoint.get_natural_key(self._cache) if natural_key is None: return None # This foreign key has unresolvable dependencies From c8288af87fb5734aaefc59643cb711c5a2f1d6bb Mon Sep 17 00:00:00 2001 From: Jeff Bradberry Date: Thu, 30 Apr 2020 17:06:48 -0400 Subject: [PATCH 66/73] Make sure we have a cached version of existing objects even if it isn't an object being directly imported. We might need it for relations of things that are being imported. --- awxkit/awxkit/api/pages/api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/awxkit/awxkit/api/pages/api.py b/awxkit/awxkit/api/pages/api.py index 28e9393462..522b749c82 100644 --- a/awxkit/awxkit/api/pages/api.py +++ b/awxkit/awxkit/api/pages/api.py @@ -175,7 +175,7 @@ class ApiV2(base.Base): def _dependent_resources(self, data): page_resource = {getattr(self, resource)._create().__item_class__: resource for resource in self.json} - data_pages = [getattr(self, resource)._create().__item_class__ for resource in data] + data_pages = [getattr(self, resource)._create().__item_class__ for resource in EXPORTABLE_RESOURCES] for page_cls in itertools.chain(*has_create.page_creation_order(*data_pages)): yield page_resource[page_cls] From 65fc2db42ff57e717cc207512c3edb337ec37a10 Mon Sep 17 00:00:00 2001 From: Jeff Bradberry Date: Thu, 30 Apr 2020 18:13:13 -0400 Subject: [PATCH 67/73] Recursively queue up related assets to be created and/or assigned --- awxkit/awxkit/api/pages/api.py | 124 ++++++++++++++++++-------------- awxkit/awxkit/api/pages/page.py | 3 + 2 files changed, 72 insertions(+), 55 deletions(-) diff --git a/awxkit/awxkit/api/pages/api.py b/awxkit/awxkit/api/pages/api.py index 522b749c82..60e6e959e4 100644 --- a/awxkit/awxkit/api/pages/api.py +++ b/awxkit/awxkit/api/pages/api.py @@ -1,5 +1,6 @@ import itertools import logging +import queue from awxkit.api.resources import resources import awxkit.exceptions as exc @@ -181,6 +182,7 @@ class ApiV2(base.Base): yield page_resource[page_cls] def _import_list(self, endpoint, assets): + log.debug("_import_list -- endpoint: %s, assets: %s", endpoint.endpoint, repr(assets)) post_fields = utils.get_post_fields(endpoint, self._cache) for asset in assets: post_data = {} @@ -214,67 +216,79 @@ class ApiV2(base.Base): self._cache.set_page(_page) - def _assign_roles(self, page, roles): - role_endpoint = page.json['related']['roles'] - for role in roles: - if 'content_object' not in role: - continue # admin role - obj_page = self._cache.get_by_natural_key(role['content_object']) - if obj_page is not None: - role_page = obj_page.get_object_role(role['name'], by_name=True) - try: - role_endpoint.post({'id': role_page['id']}) - except exc.NoContent: # desired exception on successful (dis)association - pass - else: - pass # admin role - - def _assign_related(self, _page, name, related_set): - endpoint = _page.related[name] - if isinstance(related_set, dict): # Relateds that are just json blobs, e.g. survey_spec - endpoint.post(related_set) - return - - if 'natural_key' not in related_set[0]: # It is an attach set - # Try to impedance match - related = endpoint.get(all_pages=True) - existing = {rel['id'] for rel in related.results} - for item in related_set: - rel_page = self._cache.get_by_natural_key(item) - if rel_page is None: - continue # FIXME - if rel_page['id'] in existing: - continue - try: - post_data = {'id': rel_page['id']} - endpoint.post(post_data) - except exc.NoContent: # desired exception on successful (dis)association - pass - except exc.Common as e: - log.error("Object association failed: %s.", e) - log.debug("post_data: %r", post_data) - raise - else: # It is a create set - self._import_list(endpoint, related_set) - - # FIXME: deal with pruning existing relations that do not match the import set - - def _assign_related_assets(self, assets): - for asset in assets: - _page = self._cache.get_by_natural_key(asset['natural_key']) - if _page is None: - log.error("Related object with natural key not found: %r", asset['natural_key']) - continue + # Queue up everything related to be either created or assigned. for name, S in asset.get('related', {}).items(): if not S: continue if name == 'roles': - self._assign_roles(_page, S) + self._roles.put((_page, S)) else: - self._assign_related(_page, name, S) + self._related.put((_page, name, S)) + + + def _assign_roles(self): + while True: + try: + _page, roles = self._roles.get_nowait() + self._roles.task_done() + role_endpoint = _page.json['related']['roles'] + for role in roles: + if 'content_object' not in role: + continue # admin role + obj_page = self._cache.get_by_natural_key(role['content_object']) + if obj_page is not None: + role_page = obj_page.get_object_role(role['name'], by_name=True) + try: + role_endpoint.post({'id': role_page['id']}) + except exc.NoContent: # desired exception on successful (dis)association + pass + else: + pass # admin role + except queue.Empty: + break + + def _assign_related(self): + while True: + try: + _page, name, related_set = self._related.get_nowait() + self._related.task_done() + endpoint = _page.related[name] + if isinstance(related_set, dict): # Relateds that are just json blobs, e.g. survey_spec + endpoint.post(related_set) + return + + if 'natural_key' not in related_set[0]: # It is an attach set + # Try to impedance match + related = endpoint.get(all_pages=True) + existing = {rel['id'] for rel in related.results} + for item in related_set: + rel_page = self._cache.get_by_natural_key(item) + if rel_page is None: + continue # FIXME + if rel_page['id'] in existing: + continue + try: + post_data = {'id': rel_page['id']} + endpoint.post(post_data) + log.error("endpoint: %s, id: %s", endpoint.endpoint, rel_page['id']) + except exc.NoContent: # desired exception on successful (dis)association + pass + except exc.Common as e: + log.error("Object association failed: %s.", e) + log.debug("post_data: %r", post_data) + raise + else: # It is a create set + self._cache.get_page(endpoint) + self._import_list(endpoint, related_set) + + # FIXME: deal with pruning existing relations that do not match the import set + except queue.Empty: + break def import_assets(self, data): self._cache = page.PageCache() + self._related = queue.Queue() + self._roles = queue.Queue() for resource in self._dependent_resources(data): endpoint = getattr(self, resource) @@ -283,8 +297,8 @@ class ApiV2(base.Base): self._import_list(endpoint, data.get(resource) or []) # FIXME: should we delete existing unpatched assets? - for assets in data.values(): - self._assign_related_assets(assets) + self._assign_related() + self._assign_roles() page.register_page(resources.v2, ApiV2) diff --git a/awxkit/awxkit/api/pages/page.py b/awxkit/awxkit/api/pages/page.py index 814297a4eb..338f370a90 100644 --- a/awxkit/awxkit/api/pages/page.py +++ b/awxkit/awxkit/api/pages/page.py @@ -562,6 +562,7 @@ class PageCache(object): return self.options.setdefault(url, options) def set_page(self, page): + log.debug("set_page: %s", page.endpoint) self.pages_by_url[page.endpoint] = page if getattr(page, 'NATURAL_KEY', None): natural_key = page.get_natural_key(cache=self) @@ -588,9 +589,11 @@ class PageCache(object): log.warning("This endpoint is deprecated: %s", url) return self.pages_by_url.setdefault(url, None) + log.debug("get_page: %s", page.endpoint) return self.set_page(page) def get_by_natural_key(self, natural_key): endpoint = self.pages_by_natural_key.get(utils.freeze(natural_key)) + log.debug("get_by_natural_key: %s, endpoint: %s", repr(natural_key), endpoint) if endpoint: return self.get_page(endpoint) From 1f7c1af64ec2ff69abd317db7e984403247d2e1f Mon Sep 17 00:00:00 2001 From: Jeff Bradberry Date: Fri, 1 May 2020 11:44:03 -0400 Subject: [PATCH 68/73] Fix WFJT nodes endpoint to return node pages when posting not the node list page. --- awxkit/awxkit/api/pages/api.py | 1 - awxkit/awxkit/api/pages/page.py | 4 +++- awxkit/awxkit/api/pages/workflow_job_template_nodes.py | 4 ++-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/awxkit/awxkit/api/pages/api.py b/awxkit/awxkit/api/pages/api.py index 60e6e959e4..f18a5cce1c 100644 --- a/awxkit/awxkit/api/pages/api.py +++ b/awxkit/awxkit/api/pages/api.py @@ -225,7 +225,6 @@ class ApiV2(base.Base): else: self._related.put((_page, name, S)) - def _assign_roles(self): while True: try: diff --git a/awxkit/awxkit/api/pages/page.py b/awxkit/awxkit/api/pages/page.py index 338f370a90..65a6290bb6 100644 --- a/awxkit/awxkit/api/pages/page.py +++ b/awxkit/awxkit/api/pages/page.py @@ -562,11 +562,13 @@ class PageCache(object): return self.options.setdefault(url, options) def set_page(self, page): - log.debug("set_page: %s", page.endpoint) + log.debug("set_page: %s %s", type(page), page.endpoint) self.pages_by_url[page.endpoint] = page if getattr(page, 'NATURAL_KEY', None): + log.debug("set_page has natural key fields.") natural_key = page.get_natural_key(cache=self) if natural_key is not None: + log.debug("set_page natural_key: %s", repr(natural_key)) self.pages_by_natural_key[utils.freeze(natural_key)] = page.endpoint if 'results' in page: for p in page.results: diff --git a/awxkit/awxkit/api/pages/workflow_job_template_nodes.py b/awxkit/awxkit/api/pages/workflow_job_template_nodes.py index 1b61754928..5494d6063b 100644 --- a/awxkit/awxkit/api/pages/workflow_job_template_nodes.py +++ b/awxkit/awxkit/api/pages/workflow_job_template_nodes.py @@ -123,8 +123,8 @@ class WorkflowJobTemplateNode(HasCreate, base.Base): page.register_page([resources.workflow_job_template_node, - (resources.workflow_job_template_nodes, - 'post')], + (resources.workflow_job_template_nodes, 'post'), + (resources.workflow_job_template_workflow_nodes, 'post')], WorkflowJobTemplateNode) From aeeed4d3712bd6c6a00e7efe832b9490adbf202f Mon Sep 17 00:00:00 2001 From: Jeff Bradberry Date: Fri, 8 May 2020 13:17:19 -0400 Subject: [PATCH 69/73] Add more logging --- awxkit/awxkit/api/pages/api.py | 17 +++++++++++------ awxkit/awxkit/api/pages/page.py | 14 +++++--------- awxkit/awxkit/api/pages/roles.py | 7 +++++++ awxkit/awxkit/api/utils.py | 5 +++++ 4 files changed, 28 insertions(+), 15 deletions(-) diff --git a/awxkit/awxkit/api/pages/api.py b/awxkit/awxkit/api/pages/api.py index f18a5cce1c..7615113f83 100644 --- a/awxkit/awxkit/api/pages/api.py +++ b/awxkit/awxkit/api/pages/api.py @@ -60,6 +60,7 @@ class ApiV2(base.Base): def _export(self, _page, post_fields): # Drop any (credential_type) assets that are being managed by the Tower instance. if _page.json.get('managed_by_tower'): + log.debug("%s is managed by Tower, skipping.", _page.endpoint) return None if post_fields is None: # Deprecated endpoint or insufficient permissions log.error("Object export failed: %s", _page.endpoint) @@ -78,14 +79,16 @@ class ApiV2(base.Base): rel_endpoint = self._cache.get_page(_page.related[key]) if rel_endpoint is None: # This foreign key is unreadable if post_fields[key].get('required'): - log.error("Foreign key export failed: %s", _page.related[key]) + log.error("Foreign key %r export failed for object %s.", key, _page.endpoint) return None - log.error("Foreign key export failed, setting to null: %s", _page.related[key]) + log.warning("Foreign key %r export failed for object %s, setting to null", key, _page.endpoint) continue - natural_key = rel_endpoint.get_natural_key(self._cache) - if natural_key is None: + rel_natural_key = rel_endpoint.get_natural_key(self._cache) + if rel_natural_key is None: + log.error("Unable to construct a natural key for foreign key %r of object %s.", + key, _page.endpoint) return None # This foreign key has unresolvable dependencies - fields[key] = natural_key + fields[key] = rel_natural_key related = {} for key, rel_endpoint in _page.related.items(): @@ -99,7 +102,8 @@ class ApiV2(base.Base): continue rel_post_fields = utils.get_post_fields(rel_endpoint, self._cache) - if rel_post_fields is None: # This is a read-only endpoint. + if rel_post_fields is None: + log.debug("%s is a read-only endpoint.", rel_endpoint) continue is_attach = 'id' in rel_post_fields # This is not a create-only endpoint. @@ -128,6 +132,7 @@ class ApiV2(base.Base): natural_key = _page.get_natural_key(self._cache) if natural_key is None: + log.error("Unable to construct a natural key for object %s.", _page.endpoint) return None fields['natural_key'] = natural_key diff --git a/awxkit/awxkit/api/pages/page.py b/awxkit/awxkit/api/pages/page.py index 65a6290bb6..3ee1c38490 100644 --- a/awxkit/awxkit/api/pages/page.py +++ b/awxkit/awxkit/api/pages/page.py @@ -319,27 +319,23 @@ class Page(object): return page_cls(self.connection, endpoint=endpoint).get(**kw) def get_natural_key(self, cache=None): - warn = "This object does not have a natural key: %s" - if cache is None: cache = PageCache() if not getattr(self, 'NATURAL_KEY', None): - log.warning(warn, getattr(self, 'endpoint', '')) + log.warning("This object does not have a natural key: %s", getattr(self, 'endpoint', '')) return None natural_key = {} for key in self.NATURAL_KEY: if key in self.related: related_endpoint = cache.get_page(self.related[key]) - if related_endpoint is None: - return None - natural_key[key] = related_endpoint.get_natural_key(cache=cache) + if related_endpoint is not None: + natural_key[key] = related_endpoint.get_natural_key(cache=cache) + else: + natural_key[key] = None elif key in self: natural_key[key] = self[key] - if not natural_key: - log.warning(warn, getattr(self, 'endpoint', '')) - return None natural_key['type'] = self['type'] return natural_key diff --git a/awxkit/awxkit/api/pages/roles.py b/awxkit/awxkit/api/pages/roles.py index 104af818b1..2b317dece1 100644 --- a/awxkit/awxkit/api/pages/roles.py +++ b/awxkit/awxkit/api/pages/roles.py @@ -1,9 +1,14 @@ +import logging + from awxkit.api.resources import resources from . import base from . import page +log = logging.getLogger(__name__) + + class Role(base.Base): NATURAL_KEY = ('name',) @@ -20,6 +25,8 @@ class Role(base.Base): if related_objs: related_endpoint = cache.get_page(related_objs[0]) if related_endpoint is None: + log.error("Unable to obtain content_object %s for role %s", + related_objs[0], self.endpoint) return None natural_key['content_object'] = related_endpoint.get_natural_key(cache=cache) diff --git a/awxkit/awxkit/api/utils.py b/awxkit/awxkit/api/utils.py index 355cdb0158..a3e6739b26 100644 --- a/awxkit/awxkit/api/utils.py +++ b/awxkit/awxkit/api/utils.py @@ -1,6 +1,9 @@ +import logging import re +log = logging.getLogger(__name__) + descRE = re.compile(r'^[*] `(\w+)`: [^(]*\((\w+), ([^)]+)\)') @@ -42,4 +45,6 @@ def get_post_fields(page, cache): if 'POST' in options_page.json['actions']: return options_page.json['actions']['POST'] else: + log.warning( + "Insufficient privileges on %s, inferring POST fields from description.", options_page.endpoint) return parse_description(options_page.json['description']) From 19b7c91486c6a401f0fcb82f1c755d94dbc72dca Mon Sep 17 00:00:00 2001 From: Jeff Bradberry Date: Thu, 28 May 2020 15:19:34 -0400 Subject: [PATCH 70/73] Return the changed status of imports --- awxkit/awxkit/api/pages/api.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/awxkit/awxkit/api/pages/api.py b/awxkit/awxkit/api/pages/api.py index 7615113f83..81aef61902 100644 --- a/awxkit/awxkit/api/pages/api.py +++ b/awxkit/awxkit/api/pages/api.py @@ -189,6 +189,9 @@ class ApiV2(base.Base): def _import_list(self, endpoint, assets): log.debug("_import_list -- endpoint: %s, assets: %s", endpoint.endpoint, repr(assets)) post_fields = utils.get_post_fields(endpoint, self._cache) + + changed = False + for asset in assets: post_data = {} for field, value in asset.items(): @@ -207,6 +210,7 @@ class ApiV2(base.Base): # We should only impose a default password if the resource doesn't exist. post_data.setdefault('password', 'abc123') _page = endpoint.post(post_data) + changed = True if asset['natural_key']['type'] == 'project': # When creating a project, we need to wait for its # first project update to finish so that associated @@ -214,6 +218,7 @@ class ApiV2(base.Base): _page.wait_until_completed() else: _page = _page.put(post_data) + changed = True except (exc.Common, AssertionError) as e: log.error("Object import failed: %s.", e) log.debug("post_data: %r", post_data) @@ -230,6 +235,8 @@ class ApiV2(base.Base): else: self._related.put((_page, name, S)) + return changed + def _assign_roles(self): while True: try: @@ -294,15 +301,20 @@ class ApiV2(base.Base): self._related = queue.Queue() self._roles = queue.Queue() + changed = False + for resource in self._dependent_resources(data): endpoint = getattr(self, resource) # Load up existing objects, so that we can try to update or link to them self._cache.get_page(endpoint) - self._import_list(endpoint, data.get(resource) or []) + imported = self._import_list(endpoint, data.get(resource) or []) + changed = changed or imported # FIXME: should we delete existing unpatched assets? self._assign_related() self._assign_roles() + return changed + page.register_page(resources.v2, ApiV2) From bf6e8f8e83d1caa43cf0aa1499d14b0f57cc4395 Mon Sep 17 00:00:00 2001 From: Jeff Bradberry Date: Wed, 10 Jun 2020 15:27:15 -0400 Subject: [PATCH 71/73] Prioritize membership roles since certain role grants will not be accepted by the api unless the user or team is part of the correct organization. --- awxkit/awxkit/api/pages/api.py | 116 ++++++++++++++++----------------- 1 file changed, 58 insertions(+), 58 deletions(-) diff --git a/awxkit/awxkit/api/pages/api.py b/awxkit/awxkit/api/pages/api.py index 81aef61902..fe57fe33e8 100644 --- a/awxkit/awxkit/api/pages/api.py +++ b/awxkit/awxkit/api/pages/api.py @@ -1,6 +1,5 @@ import itertools import logging -import queue from awxkit.api.resources import resources import awxkit.exceptions as exc @@ -231,75 +230,75 @@ class ApiV2(base.Base): if not S: continue if name == 'roles': - self._roles.put((_page, S)) + self._roles.append((_page, S)) else: - self._related.put((_page, name, S)) + self._related.append((_page, name, S)) return changed + def _assign_role(self, endpoint, role): + if 'content_object' not in role: + return + obj_page = self._cache.get_by_natural_key(role['content_object']) + if obj_page is None: + return + role_page = obj_page.get_object_role(role['name'], by_name=True) + try: + endpoint.post({'id': role_page['id']}) + except exc.NoContent: # desired exception on successful (dis)association + pass + + def _assign_membership(self): + for _page, roles in self._roles: + role_endpoint = _page.json['related']['roles'] + for role in roles: + if role['name'] == 'Member': + self._assign_role(role_endpoint, role) + def _assign_roles(self): - while True: - try: - _page, roles = self._roles.get_nowait() - self._roles.task_done() - role_endpoint = _page.json['related']['roles'] - for role in roles: - if 'content_object' not in role: - continue # admin role - obj_page = self._cache.get_by_natural_key(role['content_object']) - if obj_page is not None: - role_page = obj_page.get_object_role(role['name'], by_name=True) - try: - role_endpoint.post({'id': role_page['id']}) - except exc.NoContent: # desired exception on successful (dis)association - pass - else: - pass # admin role - except queue.Empty: - break + for _page, roles in self._roles: + role_endpoint = _page.json['related']['roles'] + for role in roles: + if role['name'] != 'Member': + self._assign_role(role_endpoint, role) def _assign_related(self): - while True: - try: - _page, name, related_set = self._related.get_nowait() - self._related.task_done() - endpoint = _page.related[name] - if isinstance(related_set, dict): # Relateds that are just json blobs, e.g. survey_spec - endpoint.post(related_set) - return + for _page, name, related_set in self._related: + endpoint = _page.related[name] + if isinstance(related_set, dict): # Relateds that are just json blobs, e.g. survey_spec + endpoint.post(related_set) + return - if 'natural_key' not in related_set[0]: # It is an attach set - # Try to impedance match - related = endpoint.get(all_pages=True) - existing = {rel['id'] for rel in related.results} - for item in related_set: - rel_page = self._cache.get_by_natural_key(item) - if rel_page is None: - continue # FIXME - if rel_page['id'] in existing: - continue - try: - post_data = {'id': rel_page['id']} - endpoint.post(post_data) - log.error("endpoint: %s, id: %s", endpoint.endpoint, rel_page['id']) - except exc.NoContent: # desired exception on successful (dis)association - pass - except exc.Common as e: - log.error("Object association failed: %s.", e) - log.debug("post_data: %r", post_data) - raise - else: # It is a create set - self._cache.get_page(endpoint) - self._import_list(endpoint, related_set) + if 'natural_key' not in related_set[0]: # It is an attach set + # Try to impedance match + related = endpoint.get(all_pages=True) + existing = {rel['id'] for rel in related.results} + for item in related_set: + rel_page = self._cache.get_by_natural_key(item) + if rel_page is None: + continue # FIXME + if rel_page['id'] in existing: + continue + try: + post_data = {'id': rel_page['id']} + endpoint.post(post_data) + log.error("endpoint: %s, id: %s", endpoint.endpoint, rel_page['id']) + except exc.NoContent: # desired exception on successful (dis)association + pass + except exc.Common as e: + log.error("Object association failed: %s.", e) + log.debug("post_data: %r", post_data) + raise + else: # It is a create set + self._cache.get_page(endpoint) + self._import_list(endpoint, related_set) - # FIXME: deal with pruning existing relations that do not match the import set - except queue.Empty: - break + # FIXME: deal with pruning existing relations that do not match the import set def import_assets(self, data): self._cache = page.PageCache() - self._related = queue.Queue() - self._roles = queue.Queue() + self._related = [] + self._roles = [] changed = False @@ -312,6 +311,7 @@ class ApiV2(base.Base): # FIXME: should we delete existing unpatched assets? self._assign_related() + self._assign_membership() self._assign_roles() return changed From 0fda9d2c56ab4d5c328676ff70744f963b6de3b8 Mon Sep 17 00:00:00 2001 From: Jeff Bradberry Date: Thu, 11 Jun 2020 11:17:52 -0400 Subject: [PATCH 72/73] Continue after failures to grant roles --- awxkit/awxkit/api/pages/api.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/awxkit/awxkit/api/pages/api.py b/awxkit/awxkit/api/pages/api.py index fe57fe33e8..066153bfd3 100644 --- a/awxkit/awxkit/api/pages/api.py +++ b/awxkit/awxkit/api/pages/api.py @@ -247,6 +247,9 @@ class ApiV2(base.Base): endpoint.post({'id': role_page['id']}) except exc.NoContent: # desired exception on successful (dis)association pass + except exc.Common as e: + log.error("Role assignment failed: %s.", e) + log.debug("post_data: %r", {'id': role_page['id']}) def _assign_membership(self): for _page, roles in self._roles: @@ -288,7 +291,6 @@ class ApiV2(base.Base): except exc.Common as e: log.error("Object association failed: %s.", e) log.debug("post_data: %r", post_data) - raise else: # It is a create set self._cache.get_page(endpoint) self._import_list(endpoint, related_set) From 393369adec482f73e963b73832e7e3ff8b689542 Mon Sep 17 00:00:00 2001 From: Jeff Bradberry Date: Thu, 18 Jun 2020 15:32:49 -0400 Subject: [PATCH 73/73] First cut at examples of the import and export commands --- awxkit/awxkit/cli/docs/source/examples.rst | 36 +++++++++++++++++++--- 1 file changed, 32 insertions(+), 4 deletions(-) diff --git a/awxkit/awxkit/cli/docs/source/examples.rst b/awxkit/awxkit/cli/docs/source/examples.rst index 592f60eec9..95d6ecb966 100644 --- a/awxkit/awxkit/cli/docs/source/examples.rst +++ b/awxkit/awxkit/cli/docs/source/examples.rst @@ -59,9 +59,37 @@ Importing an SSH Key --name 'My SSH Key' --user 'alice' \ --inputs '{"username": "server-login", "ssh_key_data": "@~/.ssh/id_rsa"}' -Backup/Restore --------------- +Import/Export +------------- -The AWX CLI doesn't currently have official support for backing up and restoring resources (similar to `tower-cli send` and `tower-cli receive`). +Intended to be similar to `tower-cli send` and `tower-cli receive`. -If you rely on this functionality, you should continue to use `tower-cli` at this time. +Exporting everything: + +.. code:: bash + + awx export + +Exporting everything of some particular type or types: + +.. code:: bash + + awx export --users + +Exporting a particular named resource: + +.. code:: bash + + awx export --users admin + +Exporting a resource by id: + +.. code:: bash + + awx export --users 42 + +Importing a set of resources stored as a file: + +.. code:: bash + + awx import < resources.json