External-Mirrors/awx
2
0
Fork 0
mirror of https://github.com/ansible/awx.git synced 2026-03-23 11:55:04 -02:30
Code Issues Packages Projects Releases Wiki Activity

tweaks to make data generator not fail on super large job event data sets

Browse Source 
refine formatting of job event print statement
update presets.tsv file
This commit is contained in:
AlanCoding
2017-01-30 09:17:46 -05:00
parent e9f22be216
commit efa676f3e7
2 changed files with 63 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files

30
tools/data_generators/presets.tsv
View File

@@ -1,15 +1,15 @@
resource medium Jan2017 jobs1k jobs10k jobs50k jobs100k resource small medium Jan2017 jobs1k jobs10k jobs50k jobs100k jobs200k
organizations 500 1 1 1 1 1 organizations 50 500 1 1 1 1 1 1
users 5000 3 3 3 3 3 users 500 5000 3 3 3 3 3 3
teams 500 2 2 2 2 2 teams 200 500 2 2 2 2 2 2
projects 1000 30 30 30 30 30 projects 150 1000 30 30 30 30 30 30
job_templates 2000 127 127 127 127 127 job_templates 300 2000 127 127 127 127 127 127
credentials 2000 50 50 50 50 50 credentials 150 2000 50 50 50 50 50 50
inventories 2000 6 6 6 6 6 inventories 150 2000 6 6 6 6 6 6
inventory_groups 500 15 15 15 15 15 inventory_groups 700 500 15 15 15 15 15 15
inventory_hosts 2500 15 15 15 15 15 inventory_hosts 1100 2500 15 15 15 15 15 15
wfjts 100 0 0 0 0 0 wfjts 50 100 0 0 0 0 0 0
nodes 1000 0 0 0 0 0 nodes 1000 1000 0 0 0 0 0 0
labels 1000 0 0 0 0 0 labels 1000 1000 0 0 0 0 0 0
jobs 1000 157208 1000 10000 50000 100000 jobs 2000 5000 157208 1000 10000 50000 100000 200000
job_events 1000 3370942 20000 200000 1000000 2000000 job_events 40000 100000 3370942 20000 200000 1000000 2000000 4000000
1 resource small medium Jan2017 jobs1k jobs10k jobs50k jobs100k jobs200k
2 organizations 50 500 1 1 1 1 1 1
3 users 500 5000 3 3 3 3 3 3
4 teams 200 500 2 2 2 2 2 2
5 projects 150 1000 30 30 30 30 30 30
6 job_templates 300 2000 127 127 127 127 127 127
7 credentials 150 2000 50 50 50 50 50 50
8 inventories 150 2000 6 6 6 6 6 6
9 inventory_groups 700 500 15 15 15 15 15 15
10 inventory_hosts 1100 2500 15 15 15 15 15 15
11 wfjts 50 100 0 0 0 0 0 0
12 nodes 1000 1000 0 0 0 0 0 0
13 labels 1000 1000 0 0 0 0 0 0
14 jobs 2000 1000 5000 157208 1000 10000 50000 100000 200000
15 job_events 40000 1000 100000 3370942 20000 200000 1000000 2000000 4000000

59
tools/data_generators/rbac_dummy_data_generator.py
View File

@@ -40,6 +40,11 @@ from django.db import transaction # noqa
# awx # awx
from awx.main.models import * # noqa from awx.main.models import * # noqa
from awx.main.signals import ( # noqa
emit_update_inventory_on_created_or_deleted,
emit_update_inventory_computed_fields
)
from django.db.models.signals import post_save, post_delete, m2m_changed # noqa
option_list = [ option_list = [
@@ -182,9 +187,32 @@ def mock_save(self, *args, **kwargs):
return super(PrimordialModel, self).save(*args, **kwargs) return super(PrimordialModel, self).save(*args, **kwargs)
def mock_computed_fields(self, **kwargs):
pass
PrimordialModel.save = mock_save PrimordialModel.save = mock_save
sigstat = []
sigstat.append(post_save.disconnect(emit_update_inventory_on_created_or_deleted, sender=Host))
sigstat.append(post_delete.disconnect(emit_update_inventory_on_created_or_deleted, sender=Host))
sigstat.append(post_save.disconnect(emit_update_inventory_on_created_or_deleted, sender=Group))
sigstat.append(post_delete.disconnect(emit_update_inventory_on_created_or_deleted, sender=Group))
sigstat.append(m2m_changed.disconnect(emit_update_inventory_computed_fields, sender=Group.hosts.through))
sigstat.append(m2m_changed.disconnect(emit_update_inventory_computed_fields, sender=Group.parents.through))
sigstat.append(m2m_changed.disconnect(emit_update_inventory_computed_fields, sender=Host.inventory_sources.through))
sigstat.append(m2m_changed.disconnect(emit_update_inventory_computed_fields, sender=Group.inventory_sources.through))
sigstat.append(post_save.disconnect(emit_update_inventory_on_created_or_deleted, sender=InventorySource))
sigstat.append(post_delete.disconnect(emit_update_inventory_on_created_or_deleted, sender=InventorySource))
sigstat.append(post_save.disconnect(emit_update_inventory_on_created_or_deleted, sender=Job))
sigstat.append(post_delete.disconnect(emit_update_inventory_on_created_or_deleted, sender=Job))
print ' status of signal disconnects '
print ' (True means successful disconnect)'
print str(sigstat)
startTime = datetime.now() startTime = datetime.now()
@@ -594,6 +622,8 @@ try:
print('# Adding labels to job templates') print('# Adding labels to job templates')
jt_idx = 0 jt_idx = 0
for n in spread(n_labels * 7, n_job_templates): for n in spread(n_labels * 7, n_job_templates):
if n == 0:
continue
jt = job_templates[jt_idx] jt = job_templates[jt_idx]
if not jt._is_new: if not jt._is_new:
continue continue
@@ -676,17 +706,24 @@ try:
# Check if job already has events, for idempotence # Check if job already has events, for idempotence
if not job._is_new: if not job._is_new:
continue continue
sys.stdout.write('\r Creating %d job events for job %d' % (n, job.id)) # Bulk create in chunks with maximum chunk size
sys.stdout.flush() MAX_BULK_CREATE = 100
JobEvent.objects.bulk_create([ for j in range((n / MAX_BULK_CREATE) + 1):
JobEvent( n_subgroup = MAX_BULK_CREATE
created=now(), if j == n / MAX_BULK_CREATE:
modified=now(), # on final pass, create the remainder
job=job, n_subgroup = n % MAX_BULK_CREATE
event='runner_on_ok' sys.stdout.write('\r Creating %d job events for job %d, subgroup: %d' % (n, job.id, j))
) sys.stdout.flush()
for i in range(n) JobEvent.objects.bulk_create([
]) JobEvent(
created=now(),
modified=now(),
job=job,
event='runner_on_ok'
)
for i in range(n_subgroup)
])
job_idx += 1 job_idx += 1
if n: if n:
print('') print('')