External-Mirrors/awx
2
0
Fork 0
mirror of https://github.com/ansible/awx.git synced 2026-01-16 20:30:46 -03:30
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #5105 from AlanCoding/data_gen_to_the_moon

Browse Source 
Make data generator work for larger data sets
This commit is contained in:
Alan Rominger 2017-02-03 12:34:56 -05:00 committed by GitHub
commit b822ef14c4
2 changed files with 63 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
tools/data_generators/presets.tsv
View File

@ -1,15 +1,15 @@
resource medium Jan2017 jobs1k jobs10k jobs50k jobs100k
organizations 500 1 1 1 1 1
users 5000 3 3 3 3 3
teams 500 2 2 2 2 2
projects 1000 30 30 30 30 30
job_templates 2000 127 127 127 127 127
credentials 2000 50 50 50 50 50
inventories 2000 6 6 6 6 6
inventory_groups 500 15 15 15 15 15
inventory_hosts 2500 15 15 15 15 15
wfjts 100 0 0 0 0 0
nodes 1000 0 0 0 0 0
labels 1000 0 0 0 0 0
jobs 1000 157208 1000 10000 50000 100000
job_events 1000 3370942 20000 200000 1000000 2000000
resource small medium Jan2017 jobs1k jobs10k jobs50k jobs100k jobs200k
organizations 50 500 1 1 1 1 1 1
users 500 5000 3 3 3 3 3 3
teams 200 500 2 2 2 2 2 2
projects 150 1000 30 30 30 30 30 30
job_templates 300 2000 127 127 127 127 127 127
credentials 150 2000 50 50 50 50 50 50
inventories 150 2000 6 6 6 6 6 6
inventory_groups 700 500 15 15 15 15 15 15
inventory_hosts 1100 2500 15 15 15 15 15 15
wfjts 50 100 0 0 0 0 0 0
nodes 1000 1000 0 0 0 0 0 0
labels 1000 1000 0 0 0 0 0 0
jobs 2000 5000 157208 1000 10000 50000 100000 200000
job_events 40000 100000 3370942 20000 200000 1000000 2000000 4000000
1 resource small medium Jan2017 jobs1k jobs10k jobs50k jobs100k jobs200k
2 organizations 50 500 1 1 1 1 1 1
3 users 500 5000 3 3 3 3 3 3
4 teams 200 500 2 2 2 2 2 2
5 projects 150 1000 30 30 30 30 30 30
6 job_templates 300 2000 127 127 127 127 127 127
7 credentials 150 2000 50 50 50 50 50 50
8 inventories 150 2000 6 6 6 6 6 6
9 inventory_groups 700 500 15 15 15 15 15 15
10 inventory_hosts 1100 2500 15 15 15 15 15 15
11 wfjts 50 100 0 0 0 0 0 0
12 nodes 1000 1000 0 0 0 0 0 0
13 labels 1000 1000 0 0 0 0 0 0
14 jobs 2000 1000 5000 157208 1000 10000 50000 100000 200000
15 job_events 40000 1000 100000 3370942 20000 200000 1000000 2000000 4000000

59
tools/data_generators/rbac_dummy_data_generator.py
View File

@ -40,6 +40,11 @@ from django.db import transaction # noqa
# awx
from awx.main.models import * # noqa
from awx.main.signals import ( # noqa
emit_update_inventory_on_created_or_deleted,
emit_update_inventory_computed_fields
)
from django.db.models.signals import post_save, post_delete, m2m_changed # noqa
option_list = [
@ -182,9 +187,32 @@ def mock_save(self, *args, **kwargs):
return super(PrimordialModel, self).save(*args, **kwargs)
def mock_computed_fields(self, **kwargs):
pass
PrimordialModel.save = mock_save
sigstat = []
sigstat.append(post_save.disconnect(emit_update_inventory_on_created_or_deleted, sender=Host))
sigstat.append(post_delete.disconnect(emit_update_inventory_on_created_or_deleted, sender=Host))
sigstat.append(post_save.disconnect(emit_update_inventory_on_created_or_deleted, sender=Group))
sigstat.append(post_delete.disconnect(emit_update_inventory_on_created_or_deleted, sender=Group))
sigstat.append(m2m_changed.disconnect(emit_update_inventory_computed_fields, sender=Group.hosts.through))
sigstat.append(m2m_changed.disconnect(emit_update_inventory_computed_fields, sender=Group.parents.through))
sigstat.append(m2m_changed.disconnect(emit_update_inventory_computed_fields, sender=Host.inventory_sources.through))
sigstat.append(m2m_changed.disconnect(emit_update_inventory_computed_fields, sender=Group.inventory_sources.through))
sigstat.append(post_save.disconnect(emit_update_inventory_on_created_or_deleted, sender=InventorySource))
sigstat.append(post_delete.disconnect(emit_update_inventory_on_created_or_deleted, sender=InventorySource))
sigstat.append(post_save.disconnect(emit_update_inventory_on_created_or_deleted, sender=Job))
sigstat.append(post_delete.disconnect(emit_update_inventory_on_created_or_deleted, sender=Job))
print ' status of signal disconnects '
print ' (True means successful disconnect)'
print str(sigstat)
startTime = datetime.now()
@ -594,6 +622,8 @@ try:
print('# Adding labels to job templates')
jt_idx = 0
for n in spread(n_labels * 7, n_job_templates):
if n == 0:
continue
jt = job_templates[jt_idx]
if not jt._is_new:
continue
@ -676,17 +706,24 @@ try:
# Check if job already has events, for idempotence
if not job._is_new:
continue
sys.stdout.write('\r Creating %d job events for job %d' % (n, job.id))
sys.stdout.flush()
JobEvent.objects.bulk_create([
JobEvent(
created=now(),
modified=now(),
job=job,
event='runner_on_ok'
)
for i in range(n)
])
# Bulk create in chunks with maximum chunk size
MAX_BULK_CREATE = 100
for j in range((n / MAX_BULK_CREATE) + 1):
n_subgroup = MAX_BULK_CREATE
if j == n / MAX_BULK_CREATE:
# on final pass, create the remainder
n_subgroup = n % MAX_BULK_CREATE
sys.stdout.write('\r Creating %d job events for job %d, subgroup: %d' % (n, job.id, j))
sys.stdout.flush()
JobEvent.objects.bulk_create([
JobEvent(
created=now(),
modified=now(),
job=job,
event='runner_on_ok'
)
for i in range(n_subgroup)
])
job_idx += 1
if n:
print('')