mirror of
https://github.com/ansible/awx.git
synced 2026-08-03 03:19:57 -02:30
AAP-39365 facts are unintentionally deleted when the inventory is modified during a job execution (#15910)
* Added test_jobs.py to the model unit test folder in orther to show the undesired behaviour with fact cache Signed-off-by: onetti7 <davonebat@gmail.com> * Added test_jobs.py to the model unit test folder in orther to show the undesired behaviour with fact cache Signed-off-by: onetti7 <davonebat@gmail.com> * Solved undesired behaviour with fact_cache Signed-off-by: onetti7 <davonebat@gmail.com> * Solved bug with slices Signed-off-by: onetti7 <davonebat@gmail.com> * Remove unused imports Remove now unused line of code which was commented out by the contributor Revert "Remove now unused line of code which was commented out by the contributor" This reverts commit f1a056a2356d56bc7256957a18503bd14dcfd8aa. * Add back line that had been commented out as this line makes hosts specific to the particular slice when applicable Revise private_data_dir fixture to see if it improves code coverage Checked out awx/main/tests/unit/models/test_jobs.py in devel to see if it resolves git diff issue * Fix formatting in awx/main/tests/unit/models/test_jobs.py Rename for loop from host in hosts to hosts in hosts_cahced and remove unneeded continue Revise finish_fact_cache to utilize inventory rather than hosts Remove local var hosts that was assigned but unused Revert change in start_fact_cache hosts_cached back to hosts Revise the way we are handling hosts_cached and joining the file Revert "Revise the way we are handling hosts_cached and joining the file" This reverts commit e6e3d2f09c1b79a9bce3647a72e3dad97fe0aed8. Reapply "Revise the way we are handling hosts_cached and joining the file" This reverts commit a42b7ae69133fee24d3a5f1b456d9c343d111df9. Revert some of my changes to get back to a better working state Rename for loop to host in hosts_cached and remove unneeded continue Remove jobs job.get_hosts_for_fact_cache() from post run hook, fix if statement after continue block, and revise how we are calling hosts in finish for loop Add test_invalid_host_facts to test_jobs to increase code coverage Update method signature to use hosts_cached and updated other references to hosts in finish_facts_cached Rename hosts iterator to hosts_cached to agree with naming elsewhere Revise test_invalid_host_facts to get more code coverage Revise test_invalid_host_facts to increase codecov Revise test_pre_post_run_hook_facts_deleted_sliced to ensure we are hitting the assertionerror for code cov Revise mock_inventory.hosts. to hit assert failure Add revision of hosts and facts to force failure to satisfy code cov Fix failure in test_pre_post_run_hook_facts_deleted_sliced Add back for loop to create failures and add assert to hit them Remove hosts.iterator() from both start_fact_cache and finish_fact_cache Remove unused import of Queryset to satisfy api-lint Fix typo in docstring hasnot to has not Move hosts_cached.append(host) to outer loop in start_fact_cache Add class to help support cached hosts resolving host.name issue with hosts_cached * Add live tests for ansible facts Remove fixture needed for local work only maybe Revert "Add class to help support cached hosts resolving host.name issue with hosts_cached" This reverts commit 99d998cfb9960baafe887de80bd9b01af50513ec. * Move hosts_cached.append(host) outside of try except * Move hosts_cached.append(host) to the beginning of start_fact_cache --------- Signed-off-by: onetti7 <davonebat@gmail.com> Co-authored-by: onetti7 <davonebat@gmail.com> Co-authored-by: Alan Rominger <arominge@redhat.com>
This commit is contained in:
Lila Yasin
GitHub
@@ -6,7 +6,6 @@ import logging
|
||||
|
||||
# Django
|
||||
from django.conf import settings
|
||||
from django.db.models.query import QuerySet
|
||||
from django.utils.encoding import smart_str
|
||||
from django.utils.timezone import now
|
||||
from django.db import OperationalError
|
||||
@@ -26,6 +25,7 @@ system_tracking_logger = logging.getLogger('awx.analytics.system_tracking')
|
||||
def start_fact_cache(hosts, destination, log_data, timeout=None, inventory_id=None):
|
||||
log_data['inventory_id'] = inventory_id
|
||||
log_data['written_ct'] = 0
|
||||
hosts_cached = list()
|
||||
try:
|
||||
os.makedirs(destination, mode=0o700)
|
||||
except FileExistsError:
|
||||
@@ -34,17 +34,17 @@ def start_fact_cache(hosts, destination, log_data, timeout=None, inventory_id=No
|
||||
if timeout is None:
|
||||
timeout = settings.ANSIBLE_FACT_CACHE_TIMEOUT
|
||||
|
||||
if isinstance(hosts, QuerySet):
|
||||
hosts = hosts.iterator()
|
||||
|
||||
last_filepath_written = None
|
||||
for host in hosts:
|
||||
if (not host.ansible_facts_modified) or (timeout and host.ansible_facts_modified < now() - datetime.timedelta(seconds=timeout)):
|
||||
hosts_cached.append(host)
|
||||
if not host.ansible_facts_modified or (timeout and host.ansible_facts_modified < now() - datetime.timedelta(seconds=timeout)):
|
||||
continue # facts are expired - do not write them
|
||||
|
||||
filepath = os.sep.join(map(str, [destination, host.name]))
|
||||
if not os.path.realpath(filepath).startswith(destination):
|
||||
system_tracking_logger.error('facts for host {} could not be cached'.format(smart_str(host.name)))
|
||||
continue
|
||||
|
||||
try:
|
||||
with codecs.open(filepath, 'w', encoding='utf-8') as f:
|
||||
os.chmod(f.name, 0o600)
|
||||
@@ -54,10 +54,11 @@ def start_fact_cache(hosts, destination, log_data, timeout=None, inventory_id=No
|
||||
except IOError:
|
||||
system_tracking_logger.error('facts for host {} could not be cached'.format(smart_str(host.name)))
|
||||
continue
|
||||
# make note of the time we wrote the last file so we can check if any file changed later
|
||||
|
||||
if last_filepath_written:
|
||||
return os.path.getmtime(last_filepath_written)
|
||||
return None
|
||||
return os.path.getmtime(last_filepath_written), hosts_cached
|
||||
|
||||
return None, hosts_cached
|
||||
|
||||
|
||||
def raw_update_hosts(host_list):
|
||||
@@ -88,17 +89,14 @@ def update_hosts(host_list, max_tries=5):
|
||||
msg='Inventory {inventory_id} host facts: updated {updated_ct}, cleared {cleared_ct}, unchanged {unmodified_ct}, took {delta:.3f} s',
|
||||
add_log_data=True,
|
||||
)
|
||||
def finish_fact_cache(hosts, destination, facts_write_time, log_data, job_id=None, inventory_id=None):
|
||||
def finish_fact_cache(hosts_cached, destination, facts_write_time, log_data, job_id=None, inventory_id=None):
|
||||
log_data['inventory_id'] = inventory_id
|
||||
log_data['updated_ct'] = 0
|
||||
log_data['unmodified_ct'] = 0
|
||||
log_data['cleared_ct'] = 0
|
||||
|
||||
if isinstance(hosts, QuerySet):
|
||||
hosts = hosts.iterator()
|
||||
|
||||
hosts_to_update = []
|
||||
for host in hosts:
|
||||
for host in hosts_cached:
|
||||
filepath = os.sep.join(map(str, [destination, host.name]))
|
||||
if not os.path.realpath(filepath).startswith(destination):
|
||||
system_tracking_logger.error('facts for host {} could not be cached'.format(smart_str(host.name)))
|
||||
@@ -130,6 +128,7 @@ def finish_fact_cache(hosts, destination, facts_write_time, log_data, job_id=Non
|
||||
log_data['unmodified_ct'] += 1
|
||||
else:
|
||||
# if the file goes missing, ansible removed it (likely via clear_facts)
|
||||
# if the file goes missing, but the host has not started facts, then we should not clear the facts
|
||||
host.ansible_facts = {}
|
||||
host.ansible_facts_modified = now()
|
||||
hosts_to_update.append(host)
|
||||
|
||||
Reference in New Issue
Block a user