External-Mirrors/awx
2
0
Fork 0
mirror of https://github.com/ansible/awx.git synced 2026-05-22 16:27:42 -02:30
Code Issues Packages Projects Releases Wiki Activity

Use the new PageCache to store and reuse Page.get results

Browse Source
This commit is contained in:
Jeff Bradberry
2020-04-13 17:27:29 -04:00
parent 1300d38e47
commit 14b5f63bd8
3 changed files with 60 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files

33
awxkit/awxkit/api/pages/api.py
View File

@@ -78,14 +78,13 @@ class ApiV2(base.Base):
for key in options: for key in options:
if key not in asset.related: if key not in asset.related:
continue continue
try:
# FIXME: use caching by url related_endpoint = self._cache.get_page(asset.related[key])
natural_key = asset.related[key].get().get_natural_key() if related_endpoint is None:
except exc.Forbidden: return None # This foreign key is unreadable
log.warning("This foreign key cannot be read: %s", asset.related[key]) natural_key = related_endpoint.get_natural_key(self._cache)
return None
if natural_key is None: if natural_key is None:
return None # This is an unresolvable foreign key return None # This foreign key has unresolvable dependencies
fields[key] = natural_key fields[key] = natural_key
related = {} related = {}
@@ -106,16 +105,13 @@ class ApiV2(base.Base):
else: else:
continue continue
try: data = self._cache.get_page(related_endpoint)
# FIXME: use caching by url if data is None:
data = rel.get(all_pages=True)
except exc.Forbidden:
log.warning("These related objects cannot be read: %s", related_endpoint)
continue continue
if 'results' in data: if 'results' in data:
results = ( results = (
x.get_natural_key() if by_natural_key else self._serialize_asset(x, related_options) x.get_natural_key(self._cache) if by_natural_key else self._serialize_asset(x, related_options)
for x in data.results for x in data.results
) )
related[key] = [x for x in results if x is not None] related[key] = [x for x in results if x is not None]
@@ -125,7 +121,7 @@ class ApiV2(base.Base):
if related: if related:
fields['related'] = related fields['related'] = related
natural_key = asset.get_natural_key() natural_key = asset.get_natural_key(self._cache)
if natural_key is None: if natural_key is None:
return None return None
fields['natural_key'] = natural_key fields['natural_key'] = natural_key
@@ -144,7 +140,7 @@ class ApiV2(base.Base):
pk = pk_or_name(self, resource, value) # TODO: decide whether to support multiple pk = pk_or_name(self, resource, value) # TODO: decide whether to support multiple
results = endpoint.get(id=pk).results results = endpoint.get(id=pk).results
else: else:
results = endpoint.get(all_pages=True).results results = self._cache.get_page(endpoint).results
assets = (self._serialize_asset(asset, options) for asset in results) assets = (self._serialize_asset(asset, options) for asset in results)
return [asset for asset in assets if asset is not None] return [asset for asset in assets if asset is not None]
@@ -174,7 +170,7 @@ class ApiV2(base.Base):
yield page_resource[page_cls] yield page_resource[page_cls]
def _register_page(self, page): def _register_page(self, page):
natural_key = freeze(page.get_natural_key()) natural_key = freeze(page.get_natural_key(self._cache))
# FIXME: we need to keep a reference for the case where we # FIXME: we need to keep a reference for the case where we
# don't have a natural key, so we can delete # don't have a natural key, so we can delete
if natural_key is not None: if natural_key is not None:
@@ -189,7 +185,7 @@ class ApiV2(base.Base):
if options is None: if options is None:
return return
results = endpoint.get(all_pages=True).results results = self._cache.get_page(endpoint).results
for pg in results: for pg in results:
self._register_page(pg) self._register_page(pg)
@@ -237,6 +233,7 @@ class ApiV2(base.Base):
page = endpoint.post(post_data) page = endpoint.post(post_data)
else: else:
page = page.put(post_data) page = page.put(post_data)
# FIXME: created pages need to be put in the cache
except exc.Common: except exc.Common:
log.exception("post_data: %r", post_data) log.exception("post_data: %r", post_data)
raise raise
@@ -274,7 +271,7 @@ class ApiV2(base.Base):
for item in related_set: for item in related_set:
data = {key: value for key, value in item.items() data = {key: value for key, value in item.items()
if key not in ('natural_key', 'related')} if key not in ('natural_key', 'related')}
endpoint.post(data) endpoint.post(data) # FIXME: add the page to the cache
# FIXME: deal with objects that themselves have relateds, e.g. WFJT Nodes # FIXME: deal with objects that themselves have relateds, e.g. WFJT Nodes
# FIXME: deal with pruning existing relations that do not match the import set # FIXME: deal with pruning existing relations that do not match the import set

45
awxkit/awxkit/api/pages/page.py
View File

@@ -317,9 +317,12 @@ class Page(object):
page_cls = get_registered_page(endpoint) page_cls = get_registered_page(endpoint)
return page_cls(self.connection, endpoint=endpoint).get(**kw) return page_cls(self.connection, endpoint=endpoint).get(**kw)
def get_natural_key(self): def get_natural_key(self, cache=None):
warn = "This object does not have a natural key: %s" warn = "This object does not have a natural key: %s"
if cache is None:
cache = PageCache()
if not getattr(self, 'NATURAL_KEY', None): if not getattr(self, 'NATURAL_KEY', None):
log.warning(warn, getattr(self, 'endpoint', '')) log.warning(warn, getattr(self, 'endpoint', ''))
return None return None
@@ -327,12 +330,10 @@ class Page(object):
natural_key = {} natural_key = {}
for key in self.NATURAL_KEY: for key in self.NATURAL_KEY:
if key in self.related: if key in self.related:
try: related_endpoint = cache.get_page(self.related[key])
# FIXME: use caching by url if related_endpoint is None:
natural_key[key] = self.related[key].get().get_natural_key()
except exc.Forbidden:
log.warning("This foreign key cannot be read: %s", self.related[key])
return None return None
natural_key[key] = related_endpoint.get_natural_key(cache=cache)
elif key in self: elif key in self:
natural_key[key] = self[key] natural_key[key] = self[key]
if not natural_key: if not natural_key:
@@ -401,7 +402,7 @@ class PageList(object):
def create(self, *a, **kw): def create(self, *a, **kw):
return self.__item_class__(self.connection).create(*a, **kw) return self.__item_class__(self.connection).create(*a, **kw)
def get_natural_key(self): def get_natural_key(self, cache=None):
log.warning("This object does not have a natural key: %s", getattr(self, 'endpoint', '')) log.warning("This object does not have a natural key: %s", getattr(self, 'endpoint', ''))
return None return None
@@ -536,19 +537,47 @@ class TentativePage(str):
class PageCache(object): class PageCache(object):
def __init__(self): def __init__(self):
self.options = {} self.options = {}
self.pages_by_url = {}
def get_options(self, page): def get_options(self, page):
url = page.url if isinstance(page, Page) else str(page) url = page.endpoint if isinstance(page, Page) else str(page)
if url in self.options: if url in self.options:
return self.options[url] return self.options[url]
try: try:
options = page.options() options = page.options()
except exc.Common: except exc.Common:
log.error("This endpoint raised an error: %s", url)
return self.options.setdefault(url, None) return self.options.setdefault(url, None)
warning = options.r.headers.get('Warning', '') warning = options.r.headers.get('Warning', '')
if '299' in warning and 'deprecated' in warning: if '299' in warning and 'deprecated' in warning:
log.warning("This endpoint is deprecated: %s", url)
return self.options.setdefault(url, None) return self.options.setdefault(url, None)
return self.options.setdefault(url, options) return self.options.setdefault(url, options)
def set_page(self, page):
self.pages_by_url[page.endpoint] = page
if 'results' in page:
for p in page.results:
self.set_page(p)
return page
def get_page(self, page):
url = page.endpoint if isinstance(page, Page) else str(page)
if url in self.pages_by_url:
return self.pages_by_url[url]
try:
page = page.get(all_pages=True)
except exc.Common:
log.error("This endpoint raised an error: %s", url)
return self.pages_by_url.setdefault(url, None)
warning = page.r.headers.get('Warning', '')
if '299' in warning and 'deprecated' in warning:
log.warning("This endpoint is deprecated: %s", url)
return self.pages_by_url.setdefault(url, None)
return self.set_page(page)

14
awxkit/awxkit/api/pages/roles.py
View File

@@ -9,18 +9,20 @@ class Role(base.Base):
NATURAL_KEY = ('name',) NATURAL_KEY = ('name',)
def get_natural_key(self): def get_natural_key(self, cache=None):
natural_key = super(Role, self).get_natural_key() if cache is None:
cache = page.PageCache()
natural_key = super(Role, self).get_natural_key(cache=cache)
related_objs = [ related_objs = [
related for name, related in self.related.items() related for name, related in self.related.items()
if name not in ('users', 'teams') if name not in ('users', 'teams')
] ]
if related_objs: if related_objs:
try: related_endpoint = cache.get_page(related_objs[0])
# FIXME: use caching by url if related_endpoint is None:
natural_key['content_object'] = related_objs[0].get().get_natural_key()
except exc.Forbidden:
return None return None
natural_key['content_object'] = related_endpoint.get_natural_key(cache=cache)
return natural_key return natural_key