Recursively queue up related assets to be created and/or assigned

awxkit/awxkit/api/pages/api.py

@@ -1,5 +1,6 @@
 import itertools
 import logging
+import queue
 
 from awxkit.api.resources import resources
 import awxkit.exceptions as exc
@@ -181,6 +182,7 @@ class ApiV2(base.Base):
             yield page_resource[page_cls]
 
     def _import_list(self, endpoint, assets):
+        log.debug("_import_list -- endpoint: %s, assets: %s", endpoint.endpoint, repr(assets))
         post_fields = utils.get_post_fields(endpoint, self._cache)
         for asset in assets:
             post_data = {}
@@ -214,8 +216,22 @@ class ApiV2(base.Base):
 
             self._cache.set_page(_page)
 
-    def _assign_roles(self, page, roles):
+            # Queue up everything related to be either created or assigned.
-        role_endpoint = page.json['related']['roles']
+            for name, S in asset.get('related', {}).items():
+                if not S:
+                    continue
+                if name == 'roles':
+                    self._roles.put((_page, S))
+                else:
+                    self._related.put((_page, name, S))
+
+
+    def _assign_roles(self):
+        while True:
+            try:
+                _page, roles = self._roles.get_nowait()
+                self._roles.task_done()
+                role_endpoint = _page.json['related']['roles']
                 for role in roles:
                     if 'content_object' not in role:
                         continue  # admin role
@@ -228,8 +244,14 @@ class ApiV2(base.Base):
                             pass
                     else:
                         pass  # admin role
+            except queue.Empty:
+                break
 
-    def _assign_related(self, _page, name, related_set):
+    def _assign_related(self):
+        while True:
+            try:
+                _page, name, related_set = self._related.get_nowait()
+                self._related.task_done()
                 endpoint = _page.related[name]
                 if isinstance(related_set, dict):  # Relateds that are just json blobs, e.g. survey_spec
                     endpoint.post(related_set)
@@ -248,6 +270,7 @@ class ApiV2(base.Base):
                         try:
                             post_data = {'id': rel_page['id']}
                             endpoint.post(post_data)
+                            log.error("endpoint: %s, id: %s", endpoint.endpoint, rel_page['id'])
                         except exc.NoContent:  # desired exception on successful (dis)association
                             pass
                         except exc.Common as e:
@@ -255,26 +278,17 @@ class ApiV2(base.Base):
                             log.debug("post_data: %r", post_data)
                             raise
                 else:  # It is a create set
+                    self._cache.get_page(endpoint)
                     self._import_list(endpoint, related_set)
 
                 # FIXME: deal with pruning existing relations that do not match the import set
-
+            except queue.Empty:
-    def _assign_related_assets(self, assets):
+                break
-        for asset in assets:
-            _page = self._cache.get_by_natural_key(asset['natural_key'])
-            if _page is None:
-                log.error("Related object with natural key not found: %r", asset['natural_key'])
-                continue
-            for name, S in asset.get('related', {}).items():
-                if not S:
-                    continue
-                if name == 'roles':
-                    self._assign_roles(_page, S)
-                else:
-                    self._assign_related(_page, name, S)
 
     def import_assets(self, data):
         self._cache = page.PageCache()
+        self._related = queue.Queue()
+        self._roles = queue.Queue()
 
         for resource in self._dependent_resources(data):
             endpoint = getattr(self, resource)
@@ -283,8 +297,8 @@ class ApiV2(base.Base):
             self._import_list(endpoint, data.get(resource) or [])
             # FIXME: should we delete existing unpatched assets?
 
-        for assets in data.values():
+        self._assign_related()
-            self._assign_related_assets(assets)
+        self._assign_roles()
 
 
 page.register_page(resources.v2, ApiV2)

awxkit/awxkit/api/pages/page.py

@@ -562,6 +562,7 @@ class PageCache(object):
         return self.options.setdefault(url, options)
 
     def set_page(self, page):
+        log.debug("set_page: %s", page.endpoint)
         self.pages_by_url[page.endpoint] = page
         if getattr(page, 'NATURAL_KEY', None):
             natural_key = page.get_natural_key(cache=self)
@@ -588,9 +589,11 @@ class PageCache(object):
             log.warning("This endpoint is deprecated: %s", url)
             return self.pages_by_url.setdefault(url, None)
 
+        log.debug("get_page: %s", page.endpoint)
         return self.set_page(page)
 
     def get_by_natural_key(self, natural_key):
         endpoint = self.pages_by_natural_key.get(utils.freeze(natural_key))
+        log.debug("get_by_natural_key: %s, endpoint: %s", repr(natural_key), endpoint)
         if endpoint:
             return self.get_page(endpoint)