Use the new PageCache to store and reuse Page.get results

awxkit/awxkit/api/pages/api.py

@@ -78,14 +78,13 @@ class ApiV2(base.Base):
         for key in options:
             if key not in asset.related:
                 continue
-            try:
-                # FIXME: use caching by url
-                natural_key = asset.related[key].get().get_natural_key()
-            except exc.Forbidden:
-                log.warning("This foreign key cannot be read: %s", asset.related[key])
-                return None
+
+            related_endpoint = self._cache.get_page(asset.related[key])
+            if related_endpoint is None:
+                return None  # This foreign key is unreadable
+            natural_key = related_endpoint.get_natural_key(self._cache)
             if natural_key is None:
-                return None  # This is an unresolvable foreign key
+                return None  # This foreign key has unresolvable dependencies
             fields[key] = natural_key
 
         related = {}
@@ -106,16 +105,13 @@ class ApiV2(base.Base):
             else:
                 continue
 
-            try:
-                # FIXME: use caching by url
-                data = rel.get(all_pages=True)
-            except exc.Forbidden:
-                log.warning("These related objects cannot be read: %s", related_endpoint)
+            data = self._cache.get_page(related_endpoint)
+            if data is None:
                 continue
 
             if 'results' in data:
                 results = (
-                    x.get_natural_key() if by_natural_key else self._serialize_asset(x, related_options)
+                    x.get_natural_key(self._cache) if by_natural_key else self._serialize_asset(x, related_options)
                     for x in data.results
                 )
                 related[key] = [x for x in results if x is not None]
@@ -125,7 +121,7 @@ class ApiV2(base.Base):
         if related:
             fields['related'] = related
 
-        natural_key = asset.get_natural_key()
+        natural_key = asset.get_natural_key(self._cache)
         if natural_key is None:
             return None
         fields['natural_key'] = natural_key
@@ -144,7 +140,7 @@ class ApiV2(base.Base):
             pk = pk_or_name(self, resource, value)  # TODO: decide whether to support multiple
             results = endpoint.get(id=pk).results
         else:
-            results = endpoint.get(all_pages=True).results
+            results = self._cache.get_page(endpoint).results
 
         assets = (self._serialize_asset(asset, options) for asset in results)
         return [asset for asset in assets if asset is not None]
@@ -174,7 +170,7 @@ class ApiV2(base.Base):
             yield page_resource[page_cls]
 
     def _register_page(self, page):
-        natural_key = freeze(page.get_natural_key())
+        natural_key = freeze(page.get_natural_key(self._cache))
         # FIXME: we need to keep a reference for the case where we
         # don't have a natural key, so we can delete
         if natural_key is not None:
@@ -189,7 +185,7 @@ class ApiV2(base.Base):
         if options is None:
             return
 
-        results = endpoint.get(all_pages=True).results
+        results = self._cache.get_page(endpoint).results
         for pg in results:
             self._register_page(pg)
 
@@ -237,6 +233,7 @@ class ApiV2(base.Base):
                     page = endpoint.post(post_data)
                 else:
                     page = page.put(post_data)
+                # FIXME: created pages need to be put in the cache
             except exc.Common:
                 log.exception("post_data: %r", post_data)
                 raise
@@ -274,7 +271,7 @@ class ApiV2(base.Base):
             for item in related_set:
                 data = {key: value for key, value in item.items()
                         if key not in ('natural_key', 'related')}
-                endpoint.post(data)
+                endpoint.post(data)  # FIXME: add the page to the cache
                 # FIXME: deal with objects that themselves have relateds, e.g. WFJT Nodes
 
         # FIXME: deal with pruning existing relations that do not match the import set

awxkit/awxkit/api/pages/page.py

@@ -317,9 +317,12 @@ class Page(object):
         page_cls = get_registered_page(endpoint)
         return page_cls(self.connection, endpoint=endpoint).get(**kw)
 
-    def get_natural_key(self):
+    def get_natural_key(self, cache=None):
         warn = "This object does not have a natural key: %s"
 
+        if cache is None:
+            cache = PageCache()
+
         if not getattr(self, 'NATURAL_KEY', None):
             log.warning(warn, getattr(self, 'endpoint', ''))
             return None
@@ -327,12 +330,10 @@ class Page(object):
         natural_key = {}
         for key in self.NATURAL_KEY:
             if key in self.related:
-                try:
-                    # FIXME: use caching by url
-                    natural_key[key] = self.related[key].get().get_natural_key()
-                except exc.Forbidden:
-                    log.warning("This foreign key cannot be read: %s", self.related[key])
+                related_endpoint = cache.get_page(self.related[key])
+                if related_endpoint is None:
                     return None
+                natural_key[key] = related_endpoint.get_natural_key(cache=cache)
             elif key in self:
                 natural_key[key] = self[key]
         if not natural_key:
@@ -401,7 +402,7 @@ class PageList(object):
     def create(self, *a, **kw):
         return self.__item_class__(self.connection).create(*a, **kw)
 
-    def get_natural_key(self):
+    def get_natural_key(self, cache=None):
         log.warning("This object does not have a natural key: %s", getattr(self, 'endpoint', ''))
         return None
 
@@ -536,19 +537,47 @@ class TentativePage(str):
 class PageCache(object):
     def __init__(self):
         self.options = {}
+        self.pages_by_url = {}
 
     def get_options(self, page):
-        url = page.url if isinstance(page, Page) else str(page)
+        url = page.endpoint if isinstance(page, Page) else str(page)
         if url in self.options:
             return self.options[url]
 
         try:
             options = page.options()
         except exc.Common:
+            log.error("This endpoint raised an error: %s", url)
             return self.options.setdefault(url, None)
 
         warning = options.r.headers.get('Warning', '')
         if '299' in warning and 'deprecated' in warning:
+            log.warning("This endpoint is deprecated: %s", url)
             return self.options.setdefault(url, None)
 
         return self.options.setdefault(url, options)
+
+    def set_page(self, page):
+        self.pages_by_url[page.endpoint] = page
+        if 'results' in page:
+            for p in page.results:
+                self.set_page(p)
+        return page
+
+    def get_page(self, page):
+        url = page.endpoint if isinstance(page, Page) else str(page)
+        if url in self.pages_by_url:
+            return self.pages_by_url[url]
+
+        try:
+            page = page.get(all_pages=True)
+        except exc.Common:
+            log.error("This endpoint raised an error: %s", url)
+            return self.pages_by_url.setdefault(url, None)
+
+        warning = page.r.headers.get('Warning', '')
+        if '299' in warning and 'deprecated' in warning:
+            log.warning("This endpoint is deprecated: %s", url)
+            return self.pages_by_url.setdefault(url, None)
+
+        return self.set_page(page)

awxkit/awxkit/api/pages/roles.py

@@ -9,18 +9,20 @@ class Role(base.Base):
 
     NATURAL_KEY = ('name',)
 
-    def get_natural_key(self):
-        natural_key = super(Role, self).get_natural_key()
+    def get_natural_key(self, cache=None):
+        if cache is None:
+            cache = page.PageCache()
+
+        natural_key = super(Role, self).get_natural_key(cache=cache)
         related_objs = [
             related for name, related in self.related.items()
             if name not in ('users', 'teams')
         ]
         if related_objs:
-            try:
-                # FIXME: use caching by url
-                natural_key['content_object'] = related_objs[0].get().get_natural_key()
-            except exc.Forbidden:
+            related_endpoint = cache.get_page(related_objs[0])
+            if related_endpoint is None:
                 return None
+            natural_key['content_object'] = related_endpoint.get_natural_key(cache=cache)
 
         return natural_key