Drop objects that cannot be read or do not have a natural key

don't fail hard.

awxkit/awxkit/api/pages/api.py

@@ -1,4 +1,5 @@
 import itertools
+import logging
 
 from awxkit.api.resources import resources
 import awxkit.exceptions as exc
@@ -7,6 +8,9 @@ from . import page
 from ..mixins import has_create
 
 
+log = logging.getLogger(__name__)
+
+
 EXPORTABLE_RESOURCES = [
     'users',
     'organizations',
@@ -73,6 +77,8 @@ class ApiV2(base.Base):
         if 'POST' not in options.r.headers.get('Allow', ''):
             return self._options.setdefault(url, None)
 
+        # FIXME: if POST isn't in the actions, this is a view where we
+        # don't have write permissions.  Try to do something anyway.
         return self._options.setdefault(url, options.json['actions'].get('POST', {}))
 
     # Export methods
@@ -84,53 +90,60 @@ class ApiV2(base.Base):
         if options is None:  # Deprecated endpoint or insufficient permissions
             return None
 
-        try:
+        # Note: doing asset[key] automatically parses json blob strings, which can be a problem.
-            # Note: doing asset[key] automatically parses json blob strings, which can be a problem.
+        fields = {
-            fields = {
+            key: asset.json[key] for key in options
-                key: asset.json[key] for key in options
+            if key in asset.json and key not in asset.related and key != 'id'
-                if key in asset.json and key not in asset.related and key != 'id'
+        }
-            }
+        fields['natural_key'] = asset.get_natural_key()
-            fields['natural_key'] = asset.get_natural_key()
 
-            fk_fields = {
+        for key in options:
+            if not key in asset.related:
+                continue
+            try:
                 # FIXME: use caching by url
-                key: asset.related[key].get().get_natural_key() for key in options
+                fields[key] = asset.related[key].get().get_natural_key()
-                if key in asset.related
+            except exc.Forbidden:
-            }
+                log.warning("This object cannot be read: %s", asset.related[key])
+                pass  # FIXME: what if the fk is mandatory?
 
-            related = {}
+        related = {}
-            for key, related_endpoint in asset.related.items():
+        for key, related_endpoint in asset.related.items():
-                if key in asset.json or not related_endpoint:
+            if key in asset.json or not related_endpoint:
+                continue
+            if key == 'object_roles':
+                continue  # FIXME: we should aggregate all visited roles
+
+            rel = related_endpoint._create()
+            if rel.__class__.__name__ in EXPORTABLE_RELATIONS:
+                by_natural_key = True
+                related_options = self._get_options(related_endpoint)
+                if related_options is None:
                     continue
-                if key == 'object_roles':  # FIXME
+            elif rel.__class__.__name__ in EXPORTABLE_DEPENDENT_OBJECTS:
-                    continue
+                by_natural_key, related_options = False, None
-                rel = related_endpoint._create()
+            else:
+                continue
 
-                if rel.__class__.__name__ in EXPORTABLE_RELATIONS:
+            try:
-                    by_natural_key = True
+                # FIXME: use caching by url
-                    related_options = self._get_options(related_endpoint)
+                data = rel.get(all_pages=True)
-                    if related_options is None:
+            except exc.Forbidden:
-                        continue
+                log.warning("This object cannot be read: %s", related_endpoint)
-                elif rel.__class__.__name__ in EXPORTABLE_DEPENDENT_OBJECTS:
+                continue
-                    by_natural_key, related_options = False, None
-                else:
-                    continue
 
-                data = related_endpoint.get(all_pages=True)
+            if 'results' in data:
-                if 'results' in data:
+                results = (
-                    related[key] = [
+                    x.get_natural_key() if by_natural_key else self._serialize_asset(x, related_options)
-                        x.get_natural_key() if by_natural_key else self._serialize_asset(x, related_options)
+                    for x in data.results
-                        for x in data.results
+                )
-                    ]
+                related[key] = [x for x in results if x is not None]
-                else:
+            else:
-                    related[key] = data.json
+                related[key] = data.json
-        except exc.Forbidden:
-            return None
 
-        related_fields = {'related': related} if related else {}
+        if related:
+            fields['related'] = related
 
-        fields.update(fk_fields)
-        fields.update(related_fields)
         return fields
 
     def _get_assets(self, resource, value):

awxkit/awxkit/api/pages/page.py

@@ -318,7 +318,10 @@ class Page(object):
         return page_cls(self.connection, endpoint=endpoint).get(**kw)
 
     def get_natural_key(self):
+        warn = "This object does not have a natural key: %s"
+
         if not getattr(self, 'NATURAL_KEY', None):
+            log.warning(warn, getattr(self, 'endpoint', ''))
             return None
 
         natural_key = {}
@@ -328,10 +331,12 @@ class Page(object):
                     # FIXME: use caching by url
                     natural_key[key] = self.related[key].get().get_natural_key()
                 except exc.Forbidden:
+                    log.warning("This object cannot be read: %s", getattr(self, 'endpoint', ''))
                     return None
             elif key in self:
                 natural_key[key] = self[key]
         if not natural_key:
+            log.warning(warn, getattr(self, 'endpoint', ''))
             return None
 
         natural_key['type'] = self['type']
@@ -397,6 +402,7 @@ class PageList(object):
         return self.__item_class__(self.connection).create(*a, **kw)
 
     def get_natural_key(self):
+        log.warning("This object does not have a natural key: %s", getattr(self, 'endpoint', ''))
         return None