From 1346f7b580c7cc14dc12eff5f7f5f04ed3ef16a8 Mon Sep 17 00:00:00 2001 From: AlanCoding Date: Thu, 22 Dec 2016 12:03:56 -0500 Subject: [PATCH 1/9] fixing some of most annoying problems with bulk data generator --- Makefile | 4 + .../rbac_dummy_data_generator.py | 75 +++++++++++++------ 2 files changed, 55 insertions(+), 24 deletions(-) diff --git a/Makefile b/Makefile index 89263181bc..5ac67aeab2 100644 --- a/Makefile +++ b/Makefile @@ -506,6 +506,10 @@ test_tox: # Alias existing make target so old versions run against Jekins the same way test_jenkins : test_coverage +# Make fake data +bulk_data: + $(PYTHON) tools/data_generators/rbac_dummy_data_generator.py + # l10n TASKS # -------------------------------------- diff --git a/tools/data_generators/rbac_dummy_data_generator.py b/tools/data_generators/rbac_dummy_data_generator.py index 24b1323aea..c97b0b70ab 100755 --- a/tools/data_generators/rbac_dummy_data_generator.py +++ b/tools/data_generators/rbac_dummy_data_generator.py @@ -12,16 +12,34 @@ from optparse import make_option, OptionParser # Django import django from django.utils.timezone import now -from django.contrib.auth.models import User -from django.db import transaction -# awx -from awx.main.models import * # noqa + +base_dir = os.path.abspath( # Convert into absolute path string + os.path.join( # Current file's grandparent directory + os.path.join( # Current file's parent directory + os.path.dirname( # Current file's directory + os.path.abspath(__file__) # Current file path + ), + os.pardir + ), + os.pardir + ) +) + +if base_dir not in sys.path: + sys.path.insert(1, base_dir) os.environ.setdefault("DJANGO_SETTINGS_MODULE", "awx.settings.development") # noqa django.setup() # noqa +from django.contrib.auth.models import User # noqa +from django.db import transaction # noqa + +# awx +from awx.main.models import * # noqa + + option_list = [ make_option('--organizations', action='store', type='int', default=3, help='Number of organizations to create'), @@ -114,28 +132,37 @@ try: with transaction.atomic(): with batch_role_ancestor_rebuilding(): - admin, _ = User.objects.get_or_create(username = 'admin', is_superuser=True) - org_admin, _ = User.objects.get_or_create(username = 'org_admin') - org_member, _ = User.objects.get_or_create(username = 'org_member') - prj_admin, _ = User.objects.get_or_create(username = 'prj_admin') - jt_admin, _ = User.objects.get_or_create(username = 'jt_admin') - inv_admin, _ = User.objects.get_or_create(username = 'inv_admin') + admin, created = User.objects.get_or_create(username = 'admin', is_superuser=True) + if created: + admin.is_superuser = True + admin.save() + admin.set_password('test') + admin.save() - admin.is_superuser = True - admin.save() - admin.set_password('test') - admin.save() - org_admin.set_password('test') - org_admin.save() - org_member.set_password('test') - org_member.save() - prj_admin.set_password('test') - prj_admin.save() - jt_admin.set_password('test') - jt_admin.save() - inv_admin.set_password('test') - inv_admin.save() + org_admin, created = User.objects.get_or_create(username = 'org_admin') + if created: + org_admin.set_password('test') + org_admin.save() + org_member, created = User.objects.get_or_create(username = 'org_member') + if created: + org_member.set_password('test') + org_member.save() + + prj_admin, created = User.objects.get_or_create(username = 'prj_admin') + if created: + prj_admin.set_password('test') + prj_admin.save() + + jt_admin, created = User.objects.get_or_create(username = 'jt_admin') + if created: + jt_admin.set_password('test') + jt_admin.save() + + inv_admin, created = User.objects.get_or_create(username = 'inv_admin') + if created: + inv_admin.set_password('test') + inv_admin.save() print('# Creating %d organizations' % n_organizations) From c3aaccc0d47dd1e9610801eea0f932962ea9e7eb Mon Sep 17 00:00:00 2001 From: AlanCoding Date: Thu, 22 Dec 2016 13:53:19 -0500 Subject: [PATCH 2/9] make it so that repeated runs do not cause problems --- Makefile | 3 + .../rbac_dummy_data_generator.py | 64 ++++++++++--------- 2 files changed, 37 insertions(+), 30 deletions(-) diff --git a/Makefile b/Makefile index 5ac67aeab2..41e2bb70cc 100644 --- a/Makefile +++ b/Makefile @@ -508,6 +508,9 @@ test_jenkins : test_coverage # Make fake data bulk_data: + @if [ "$(VENV_BASE)" ]; then \ + . $(VENV_BASE)/tower/bin/activate; \ + fi; \ $(PYTHON) tools/data_generators/rbac_dummy_data_generator.py # l10n TASKS diff --git a/tools/data_generators/rbac_dummy_data_generator.py b/tools/data_generators/rbac_dummy_data_generator.py index c97b0b70ab..cdb342a9d6 100755 --- a/tools/data_generators/rbac_dummy_data_generator.py +++ b/tools/data_generators/rbac_dummy_data_generator.py @@ -169,7 +169,7 @@ try: for i in xrange(n_organizations): sys.stdout.write('\r%d ' % (i + 1)) sys.stdout.flush() - org = Organization.objects.create(name='%s Organization %d' % (prefix, i)) + org, _ = Organization.objects.get_or_create(name='%s Organization %d' % (prefix, i)) organizations.append(org) if i == 0: org.admin_role.members.add(org_admin) @@ -189,7 +189,7 @@ try: user_id = ids['user'] sys.stdout.write('\r Assigning %d to %s: %d ' % (n, organizations[org_idx].name, i+ 1)) sys.stdout.flush() - user = User.objects.create(username='%suser-%d' % (prefix, user_id)) + user, _ = User.objects.get_or_create(username='%suser-%d' % (prefix, user_id)) organizations[org_idx].member_role.members.add(user) users.append(user) org_idx += 1 @@ -204,7 +204,7 @@ try: team_id = ids['team'] sys.stdout.write('\r Assigning %d to %s: %d ' % (n, org.name, i+ 1)) sys.stdout.flush() - team = Team.objects.create(name='%s Team %d Org %d' % (prefix, team_id, org_idx), organization=org) + team, _ = Team.objects.get_or_create(name='%s Team %d Org %d' % (prefix, team_id, org_idx), organization=org) teams.append(team) org_idx += 1 print('') @@ -240,7 +240,7 @@ try: sys.stdout.write('\r %d ' % (ids['credential'])) sys.stdout.flush() credential_id = ids['credential'] - credential = Credential.objects.create(name='%s Credential %d User %d' % (prefix, credential_id, user_idx)) + credential, _ = Credential.objects.get_or_create(name='%s Credential %d User %d' % (prefix, credential_id, user_idx)) credential.admin_role.members.add(user) credentials.append(credential) user_idx += 1 @@ -256,7 +256,7 @@ try: sys.stdout.write('\r %d ' % (ids['credential'] - starting_credential_id)) sys.stdout.flush() credential_id = ids['credential'] - credential = Credential.objects.create(name='%s Credential %d team %d' % (prefix, credential_id, team_idx)) + credential, _ = Credential.objects.get_or_create(name='%s Credential %d team %d' % (prefix, credential_id, team_idx)) credential.admin_role.parents.add(team.member_role) credentials.append(credential) team_idx += 1 @@ -271,7 +271,7 @@ try: project_id = ids['project'] sys.stdout.write('\r Assigning %d to %s: %d ' % (n, org.name, i+ 1)) sys.stdout.flush() - project = Project.objects.create(name='%s Project %d Org %d' % (prefix, project_id, org_idx), organization=org) + project, _ = Project.objects.get_or_create(name='%s Project %d Org %d' % (prefix, project_id, org_idx), organization=org) projects.append(project) if org_idx == 0 and i == 0: project.admin_role.members.add(prj_admin) @@ -289,7 +289,7 @@ try: inventory_id = ids['inventory'] sys.stdout.write('\r Assigning %d to %s: %d ' % (n, org.name, i+ 1)) sys.stdout.flush() - inventory = Inventory.objects.create(name='%s Inventory %d Org %d' % (prefix, inventory_id, org_idx), organization=org) + inventory, _ = Inventory.objects.get_or_create(name='%s Inventory %d Org %d' % (prefix, inventory_id, org_idx), organization=org) inventories.append(inventory) if org_idx == 0 and i == 0: inventory.admin_role.members.add(inv_admin) @@ -308,7 +308,7 @@ try: group_id = ids['group'] sys.stdout.write('\r Assigning %d to %s: %d ' % (n, inventory.name, i+ 1)) sys.stdout.flush() - group = Group.objects.create( + group, _ = Group.objects.get_or_create( name='%s Group %d Inventory %d' % (prefix, group_id, inv_idx), inventory=inventory, ) @@ -335,7 +335,7 @@ try: host_id = ids['host'] sys.stdout.write('\r Assigning %d to %s: %d ' % (n, group.name, i+ 1)) sys.stdout.flush() - host = Host.objects.create(name='%s.host-%06d.group-%05d.dummy' % (prefix, host_id, group_idx), inventory=group.inventory) + host, _ = Host.objects.get_or_create(name='%s.host-%06d.group-%05d.dummy' % (prefix, host_id, group_idx), inventory=group.inventory) # Add the host to up to 3 groups host.groups.add(group) for m in range(2): @@ -363,7 +363,7 @@ try: if org_inv_count > 0: inventory = project.organization.inventories.all()[inv_idx % org_inv_count] - job_template = JobTemplate.objects.create( + job_template, _ = JobTemplate.objects.get_or_create( name='%s Job Template %d Project %d' % (prefix, job_template_id, project_idx), inventory=inventory, project=project, @@ -383,19 +383,21 @@ try: for i in range(n): sys.stdout.write('\r Assigning %d to %s: %d ' % (n, job_template.name, i+ 1)) sys.stdout.flush() - job = Job.objects.create(job_template=job_template) + job, _ = Job.objects.get_or_create(job_template=job_template) + job._is_new = _ jobs.append(job) - if job_template.inventory: - inv_groups = [g for g in job_template.inventory.groups.all()] - if len(inv_groups): - JobHostSummary.objects.bulk_create([ - JobHostSummary( - job=job, host=h, host_name=h.name, processed=1, - created=now(), modified=now() - ) - for h in inv_groups[group_idx % len(inv_groups)].hosts.all()[:100] - ]) + with transaction.atomic(): + if job_template.inventory: + inv_groups = [g for g in job_template.inventory.groups.all()] + if len(inv_groups) and job._is_new: + JobHostSummary.objects.bulk_create([ + JobHostSummary( + job=job, host=h, host_name=h.name, processed=1, + created=now(), modified=now() + ) + for h in inv_groups[group_idx % len(inv_groups)].hosts.all()[:100] + ]) group_idx += 1 job_template_idx += 1 if n: @@ -407,15 +409,17 @@ try: job = jobs[job_idx] sys.stdout.write('\r Creating %d job events for job %d' % (n, job.id)) sys.stdout.flush() - JobEvent.objects.bulk_create([ - JobEvent( - created=now(), - modified=now(), - job=job, - event='runner_on_ok' - ) - for i in range(n) - ]) + # Check if job already has events, for idempotence + if job._is_new: + JobEvent.objects.bulk_create([ + JobEvent( + created=now(), + modified=now(), + job=job, + event='runner_on_ok' + ) + for i in range(n) + ]) job_idx += 1 if n: print('') From be97f2ca7454fa1e35f394a147c61b99cc533d64 Mon Sep 17 00:00:00 2001 From: AlanCoding Date: Thu, 22 Dec 2016 14:37:02 -0500 Subject: [PATCH 3/9] generators plus WFJT nodes added to data generator --- .../rbac_dummy_data_generator.py | 78 ++++++++++++++++++- 1 file changed, 77 insertions(+), 1 deletion(-) diff --git a/tools/data_generators/rbac_dummy_data_generator.py b/tools/data_generators/rbac_dummy_data_generator.py index cdb342a9d6..d08c083805 100755 --- a/tools/data_generators/rbac_dummy_data_generator.py +++ b/tools/data_generators/rbac_dummy_data_generator.py @@ -59,6 +59,10 @@ option_list = [ help='Number of credentials to create'), make_option('--inventory-hosts', action='store', type='int', default=40, help='number of credentials to create'), + make_option('--wfjts', action='store', type='int', default=15, + help='number of workflow job templates to create'), + make_option('--nodes', action='store', type='int', default=200, + help='number of workflow job template nodes to create'), make_option('--jobs', action='store', type='int', default=200, help='number of job entries to create'), make_option('--job-events', action='store', type='int', default=500, @@ -84,6 +88,8 @@ n_credentials = int(options['credentials']) n_inventories = int(options['inventories']) n_inventory_groups = int(options['inventory_groups']) n_inventory_hosts = int(options['inventory_hosts']) +n_wfjts = int(options['wfjts']) +n_nodes = int(options['nodes']) n_jobs = int(options['jobs']) n_job_events = int(options['job_events']) prefix = options['prefix'] @@ -97,6 +103,8 @@ credentials = [] inventories = [] inventory_groups = [] inventory_hosts = [] +wfjts = [] +nodes = [] jobs = [] #job_events = [] @@ -122,6 +130,17 @@ def spread(n, m): ids = defaultdict(lambda: 0) +bulk_data_description = 'Made by the Tower bulk-data creator script' + + +# function to cycle through a list +def yield_choice(alist): + ix = 0 + while True: + ix += 1 + if ix >= len(alist): + ix = 0 + yield alist[ix] class Rollback(Exception): @@ -195,6 +214,11 @@ try: org_idx += 1 print('') + creator_gen = yield_choice(users) + for i in range(6): + next(creator_gen) + modifier_gen = yield_choice(users) + print('# Creating %d teams' % n_teams) org_idx = 0 for n in spread(n_teams, n_organizations): @@ -204,7 +228,11 @@ try: team_id = ids['team'] sys.stdout.write('\r Assigning %d to %s: %d ' % (n, org.name, i+ 1)) sys.stdout.flush() - team, _ = Team.objects.get_or_create(name='%s Team %d Org %d' % (prefix, team_id, org_idx), organization=org) + team, _ = Team.objects.get_or_create( + name='%s Team %d Org %d' % (prefix, team_id, org_idx), organization=org, + defaults=dict(created_by=next(creator_gen), + modified_by=next(modifier_gen)) + ) teams.append(team) org_idx += 1 print('') @@ -375,6 +403,54 @@ try: project_idx += 1 print('') + print('# Creating %d Workflow Job Templates' % n_wfjts) + org_idx = 0 + for n in spread(n_wfjts, n_organizations): + org = organizations[org_idx] + for i in range(n): + ids['wfjts'] += 1 + wfjt_id = ids['wfjts'] + sys.stdout.write('\r Assigning %d to %s: %d ' % (n, org.name, i+ 1)) + sys.stdout.flush() + wfjt, _ = WorkflowJobTemplate.objects.get_or_create( + name='%s WFJT %d Org %d' % (prefix, wfjt_id, org_idx), + description=bulk_data_description, + organization=org + ) + wfjts.append(wfjt) + org_idx += 1 + print('') + + print('# Creating %d Workflow Job Template nodes' % n_nodes) + wfjt_idx = 0 + for n in spread(n_nodes, n_wfjts): + wfjt = wfjts[wfjt_idx] + jt_gen = yield_choice(job_templates) + inv_gen = yield_choice(inventories) + cred_gen = yield_choice(credentials) + for i in range(n): + ids['nodes'] += 1 + node_id = ids['nodes'] + sys.stdout.write('\r Assigning %d to %s: %d ' % (n, wfjt.name, i+ 1)) + sys.stdout.flush() + kwargs = dict( + workflow_job_template=wfjt, + unified_job_template=next(jt_gen), + modified=now() + ) + if i % 2 == 0: + # only apply inventories for every other node + kwargs['inventory'] = next(inv_gen) + if i % 3 == 0: + # only apply prompted credential every 3rd node + kwargs['credential'] = next(cred_gen) + node, _ = WorkflowJobTemplateNode.objects.get_or_create( + **kwargs + ) + wfjts.append(wfjt) + wfjt_idx += 1 + print('') + print('# Creating %d jobs' % n_jobs) group_idx = 0 job_template_idx = 0 From 9d8b1b4a125659c7f6f37456f0e1f58bdd0280b5 Mon Sep 17 00:00:00 2001 From: AlanCoding Date: Sat, 24 Dec 2016 23:17:09 -0500 Subject: [PATCH 4/9] add WFJT node networks --- .../rbac_dummy_data_generator.py | 35 ++++++++++++++++--- 1 file changed, 31 insertions(+), 4 deletions(-) diff --git a/tools/data_generators/rbac_dummy_data_generator.py b/tools/data_generators/rbac_dummy_data_generator.py index d08c083805..49b2300f2d 100755 --- a/tools/data_generators/rbac_dummy_data_generator.py +++ b/tools/data_generators/rbac_dummy_data_generator.py @@ -130,17 +130,17 @@ def spread(n, m): ids = defaultdict(lambda: 0) -bulk_data_description = 'Made by the Tower bulk-data creator script' +bulk_data_description = 'From Tower bulk-data script' # function to cycle through a list def yield_choice(alist): ix = 0 while True: + yield alist[ix] ix += 1 if ix >= len(alist): ix = 0 - yield alist[ix] class Rollback(Exception): @@ -274,6 +274,8 @@ try: user_idx += 1 print('') + credential_gen = yield_choice(credentials) + print('# Creating %d credentials for teams' % (n_credentials // 2)) team_idx = 0 starting_credential_id = ids['credential'] @@ -395,6 +397,7 @@ try: name='%s Job Template %d Project %d' % (prefix, job_template_id, project_idx), inventory=inventory, project=project, + credential=next(credential_gen) ) job_templates.append(job_template) inv_idx += 1 @@ -428,6 +431,8 @@ try: jt_gen = yield_choice(job_templates) inv_gen = yield_choice(inventories) cred_gen = yield_choice(credentials) + parent_idx = 0 + wfjt_nodes = [] for i in range(n): ids['nodes'] += 1 node_id = ids['nodes'] @@ -447,7 +452,23 @@ try: node, _ = WorkflowJobTemplateNode.objects.get_or_create( **kwargs ) - wfjts.append(wfjt) + # nodes.append(node) + wfjt_nodes.append(node) + if i <= 3: + continue + parent_node = wfjt_nodes[parent_idx] + if parent_node.workflow_job_template != node.workflow_job_template: + raise Exception("Programming error, associating nodes in different workflows") + elif parent_node == node: + raise Exception("error, self association") + if parent_idx % 2 == 0: + parent_node.always_nodes.add(node) + else: + if (i + 1) % 3 == 0: + parent_node.failure_nodes.add(node) + else: + parent_node.success_nodes.add(node) + parent_idx = (parent_idx + 7) % len(wfjt_nodes) wfjt_idx += 1 print('') @@ -459,7 +480,13 @@ try: for i in range(n): sys.stdout.write('\r Assigning %d to %s: %d ' % (n, job_template.name, i+ 1)) sys.stdout.flush() - job, _ = Job.objects.get_or_create(job_template=job_template) + job_stat = 'successful' + if i % 4 == 0: + job_stat = 'failed' + elif i % 3 == 0: + job_stat = 'canceled' + job, _ = Job.objects.get_or_create( + job_template=job_template, status=job_stat) job._is_new = _ jobs.append(job) From bdfa29b7787df9e7221c5991b479cebc9efd7bf0 Mon Sep 17 00:00:00 2001 From: AlanCoding Date: Sun, 25 Dec 2016 18:45:58 -0500 Subject: [PATCH 5/9] apply created_by and other defaults to all --- .../rbac_dummy_data_generator.py | 76 ++++++++++++++++--- 1 file changed, 66 insertions(+), 10 deletions(-) diff --git a/tools/data_generators/rbac_dummy_data_generator.py b/tools/data_generators/rbac_dummy_data_generator.py index 49b2300f2d..2d037454cd 100755 --- a/tools/data_generators/rbac_dummy_data_generator.py +++ b/tools/data_generators/rbac_dummy_data_generator.py @@ -147,6 +147,15 @@ class Rollback(Exception): pass +# Normally the modified_by field is populated by the crum library automatically, +# but since this is ran outside the request-response cycle that won't work. +# It is disaled here. +def mock_save(self, *args, **kwargs): + return super(PrimordialModel, self).save(*args, **kwargs) + +PrimordialModel.save = mock_save + + try: with transaction.atomic(): @@ -268,7 +277,12 @@ try: sys.stdout.write('\r %d ' % (ids['credential'])) sys.stdout.flush() credential_id = ids['credential'] - credential, _ = Credential.objects.get_or_create(name='%s Credential %d User %d' % (prefix, credential_id, user_idx)) + credential, _ = Credential.objects.get_or_create( + name='%s Credential %d User %d' % (prefix, credential_id, user_idx), + defaults=dict(created_by=next(creator_gen), + modified_by=next(modifier_gen)), + kind='ssh' + ) credential.admin_role.members.add(user) credentials.append(credential) user_idx += 1 @@ -286,7 +300,12 @@ try: sys.stdout.write('\r %d ' % (ids['credential'] - starting_credential_id)) sys.stdout.flush() credential_id = ids['credential'] - credential, _ = Credential.objects.get_or_create(name='%s Credential %d team %d' % (prefix, credential_id, team_idx)) + credential, _ = Credential.objects.get_or_create( + name='%s Credential %d team %d' % (prefix, credential_id, team_idx), + defaults=dict(created_by=next(creator_gen), + modified_by=next(modifier_gen)), + kind='ssh' + ) credential.admin_role.parents.add(team.member_role) credentials.append(credential) team_idx += 1 @@ -301,7 +320,21 @@ try: project_id = ids['project'] sys.stdout.write('\r Assigning %d to %s: %d ' % (n, org.name, i+ 1)) sys.stdout.flush() - project, _ = Project.objects.get_or_create(name='%s Project %d Org %d' % (prefix, project_id, org_idx), organization=org) + project, _ = Project.objects.get_or_create( + name='%s Project %d Org %d' % (prefix, project_id, org_idx), + organization=org, + defaults=dict(created_by=next(creator_gen), + modified_by=next(modifier_gen)), + scm_url='https://github.com/jlaska/ansible-playbooks.git', + scm_type='git', + playbook_files=[ + "check.yml", "debug-50.yml", "debug.yml", "debug2.yml", + "debug_extra_vars.yml", "dynamic_inventory.yml", + "environ_test.yml", "fail_unless.yml", "pass_unless.yml", + "pause.yml", "ping-20.yml", "ping.yml", + "setfact_50.yml", "vault.yml" + ] + ) projects.append(project) if org_idx == 0 and i == 0: project.admin_role.members.add(prj_admin) @@ -319,7 +352,13 @@ try: inventory_id = ids['inventory'] sys.stdout.write('\r Assigning %d to %s: %d ' % (n, org.name, i+ 1)) sys.stdout.flush() - inventory, _ = Inventory.objects.get_or_create(name='%s Inventory %d Org %d' % (prefix, inventory_id, org_idx), organization=org) + inventory, _ = Inventory.objects.get_or_create( + name='%s Inventory %d Org %d' % (prefix, inventory_id, org_idx), + organization=org, + defaults=dict(created_by=next(creator_gen), + modified_by=next(modifier_gen)), + variables='{"ansible_connection": "local"}' + ) inventories.append(inventory) if org_idx == 0 and i == 0: inventory.admin_role.members.add(inv_admin) @@ -341,6 +380,8 @@ try: group, _ = Group.objects.get_or_create( name='%s Group %d Inventory %d' % (prefix, group_id, inv_idx), inventory=inventory, + defaults=dict(created_by=next(creator_gen), + modified_by=next(modifier_gen)) ) # Have each group have up to 3 parent groups for parent_n in range(3): @@ -365,7 +406,12 @@ try: host_id = ids['host'] sys.stdout.write('\r Assigning %d to %s: %d ' % (n, group.name, i+ 1)) sys.stdout.flush() - host, _ = Host.objects.get_or_create(name='%s.host-%06d.group-%05d.dummy' % (prefix, host_id, group_idx), inventory=group.inventory) + host, _ = Host.objects.get_or_create( + name='%s.host-%06d.group-%05d.dummy' % (prefix, host_id, group_idx), + inventory=group.inventory, + defaults=dict(created_by=next(creator_gen), + modified_by=next(modifier_gen)) + ) # Add the host to up to 3 groups host.groups.add(group) for m in range(2): @@ -392,12 +438,20 @@ try: org_inv_count = project.organization.inventories.count() if org_inv_count > 0: inventory = project.organization.inventories.all()[inv_idx % org_inv_count] + extra_kwargs = {} + if ids['job_template'] % 5 == 0: + extra_kwargs['cloud_credential'] = next(credential_gen) + if ids['job_template'] % 7 == 0: + extra_kwargs['network_credential'] = next(credential_gen) job_template, _ = JobTemplate.objects.get_or_create( name='%s Job Template %d Project %d' % (prefix, job_template_id, project_idx), inventory=inventory, project=project, - credential=next(credential_gen) + credential=next(credential_gen), + defaults=dict(created_by=next(creator_gen), + modified_by=next(modifier_gen)), + **extra_kwargs ) job_templates.append(job_template) inv_idx += 1 @@ -418,7 +472,9 @@ try: wfjt, _ = WorkflowJobTemplate.objects.get_or_create( name='%s WFJT %d Org %d' % (prefix, wfjt_id, org_idx), description=bulk_data_description, - organization=org + organization=org, + defaults=dict(created_by=next(creator_gen), + modified_by=next(modifier_gen)) ) wfjts.append(wfjt) org_idx += 1 @@ -440,8 +496,7 @@ try: sys.stdout.flush() kwargs = dict( workflow_job_template=wfjt, - unified_job_template=next(jt_gen), - modified=now() + unified_job_template=next(jt_gen) ) if i % 2 == 0: # only apply inventories for every other node @@ -450,7 +505,8 @@ try: # only apply prompted credential every 3rd node kwargs['credential'] = next(cred_gen) node, _ = WorkflowJobTemplateNode.objects.get_or_create( - **kwargs + **kwargs, + defaults=dict(modified=now()) ) # nodes.append(node) wfjt_nodes.append(node) From 8f3041d8b4bd40b56b2fbcdfb0d6e0926ad7bd10 Mon Sep 17 00:00:00 2001 From: AlanCoding Date: Sun, 25 Dec 2016 19:42:56 -0500 Subject: [PATCH 6/9] add creation of labels --- .../rbac_dummy_data_generator.py | 51 ++++++++++++++++--- 1 file changed, 44 insertions(+), 7 deletions(-) diff --git a/tools/data_generators/rbac_dummy_data_generator.py b/tools/data_generators/rbac_dummy_data_generator.py index 2d037454cd..42d02f6e56 100755 --- a/tools/data_generators/rbac_dummy_data_generator.py +++ b/tools/data_generators/rbac_dummy_data_generator.py @@ -63,6 +63,8 @@ option_list = [ help='number of workflow job templates to create'), make_option('--nodes', action='store', type='int', default=200, help='number of workflow job template nodes to create'), + make_option('--labels', action='store', type='int', default=200, + help='labels to create, will associate 10x as many'), make_option('--jobs', action='store', type='int', default=200, help='number of job entries to create'), make_option('--job-events', action='store', type='int', default=500, @@ -90,6 +92,7 @@ n_inventory_groups = int(options['inventory_groups']) n_inventory_hosts = int(options['inventory_hosts']) n_wfjts = int(options['wfjts']) n_nodes = int(options['nodes']) +n_labels = int(options['labels']) n_jobs = int(options['jobs']) n_job_events = int(options['job_events']) prefix = options['prefix'] @@ -105,6 +108,7 @@ inventory_groups = [] inventory_hosts = [] wfjts = [] nodes = [] +labels = [] jobs = [] #job_events = [] @@ -207,6 +211,7 @@ try: org.member_role.members.add(jt_admin) org.member_role.members.add(inv_admin) + organization_gen = yield_choice(organizations) print('') print('# Creating %d users' % n_users) @@ -324,7 +329,7 @@ try: name='%s Project %d Org %d' % (prefix, project_id, org_idx), organization=org, defaults=dict(created_by=next(creator_gen), - modified_by=next(modifier_gen)), + modified_by=next(modifier_gen), scm_url='https://github.com/jlaska/ansible-playbooks.git', scm_type='git', playbook_files=[ @@ -333,7 +338,7 @@ try: "environ_test.yml", "fail_unless.yml", "pass_unless.yml", "pause.yml", "ping-20.yml", "ping.yml", "setfact_50.yml", "vault.yml" - ] + ]) ) projects.append(project) if org_idx == 0 and i == 0: @@ -476,6 +481,7 @@ try: defaults=dict(created_by=next(creator_gen), modified_by=next(modifier_gen)) ) + wfjt._is_new = _ wfjts.append(wfjt) org_idx += 1 print('') @@ -484,6 +490,8 @@ try: wfjt_idx = 0 for n in spread(n_nodes, n_wfjts): wfjt = wfjts[wfjt_idx] + if not wfjt._is_new: + continue jt_gen = yield_choice(job_templates) inv_gen = yield_choice(inventories) cred_gen = yield_choice(credentials) @@ -496,7 +504,8 @@ try: sys.stdout.flush() kwargs = dict( workflow_job_template=wfjt, - unified_job_template=next(jt_gen) + unified_job_template=next(jt_gen), + modified=now() ) if i % 2 == 0: # only apply inventories for every other node @@ -505,8 +514,7 @@ try: # only apply prompted credential every 3rd node kwargs['credential'] = next(cred_gen) node, _ = WorkflowJobTemplateNode.objects.get_or_create( - **kwargs, - defaults=dict(modified=now()) + **kwargs ) # nodes.append(node) wfjt_nodes.append(node) @@ -528,6 +536,35 @@ try: wfjt_idx += 1 print('') + print('# Creating %d Labels' % n_labels) + org_idx = 0 + for n in spread(n_labels, n_organizations): + org = organizations[org_idx] + for i in range(n): + ids['labels'] += 1 + label_id = ids['labels'] + sys.stdout.write('\r Assigning %d to %s: %d ' % (n, org.name, i + 1)) + sys.stdout.flush() + label, _ = Label.objects.get_or_create( + name='%s Label %d Org %d' % (prefix, label_id, org_idx), + organization=org, + defaults=dict(created_by=next(creator_gen), + modified_by=next(modifier_gen)) + ) + labels.append(label) + org_idx += 1 + print('') + label_gen = yield_choice(labels) + + print('# Adding labels to job templates') + jt_idx = 0 + for n in spread(n_labels*7, n_job_templates): + jt = job_templates[jt_idx] + print(' Giving %d labels to %s JT' % (n, jt.name)) + for i in range(n): + jt.labels.add(next(label_gen)) + jt_idx += 1 + print('# Creating %d jobs' % n_jobs) group_idx = 0 job_template_idx = 0 @@ -537,9 +574,9 @@ try: sys.stdout.write('\r Assigning %d to %s: %d ' % (n, job_template.name, i+ 1)) sys.stdout.flush() job_stat = 'successful' - if i % 4 == 0: + if len(jobs) % 4 == 0: job_stat = 'failed' - elif i % 3 == 0: + elif len(jobs) % 11 == 0: job_stat = 'canceled' job, _ = Job.objects.get_or_create( job_template=job_template, status=job_stat) From ccf9fe4ecb7c597dcffc2513d21a0ca55e3ba111 Mon Sep 17 00:00:00 2001 From: AlanCoding Date: Mon, 26 Dec 2016 22:40:32 -0500 Subject: [PATCH 7/9] last job, more cleanup, and start work on preset data sets --- tools/data_generators/load_presets.py | 34 +++++++++ tools/data_generators/presets.tsv | 15 ++++ .../rbac_dummy_data_generator.py | 71 ++++++++++++++----- 3 files changed, 102 insertions(+), 18 deletions(-) create mode 100644 tools/data_generators/load_presets.py create mode 100644 tools/data_generators/presets.tsv diff --git a/tools/data_generators/load_presets.py b/tools/data_generators/load_presets.py new file mode 100644 index 0000000000..34561e8fa6 --- /dev/null +++ b/tools/data_generators/load_presets.py @@ -0,0 +1,34 @@ +import os + +presets_filename = os.path.abspath(os.path.join( + os.path.dirname(os.path.abspath(__file__)), + 'presets.tsv')) + +with open(presets_filename) as f: + r = f.read() + +print r + +lines = r.split('\n') +keys = lines[0].split('\t')[1:] + +preset = 'medium' + +col = None +for i, key in enumerate(keys): + if key == preset: + col = i + break + +if col is None: + raise Exception('Preset %s data set not found, options are %s' % (preset, keys)) + + +options = {} + +for line in lines[1:]: + cols = line.split('\t') + options[cols[0]] = cols[i+1] + +print ' options ' +print options diff --git a/tools/data_generators/presets.tsv b/tools/data_generators/presets.tsv new file mode 100644 index 0000000000..f0315eb702 --- /dev/null +++ b/tools/data_generators/presets.tsv @@ -0,0 +1,15 @@ +resource medium +organizations 500 +users 5000 +teams 500 +projects 1000 +job-templates 2000 +credentials 2000 +inventories 2000 +inventory-groups 500 +inventory-hosts 2500 +wfjts 100 +nodes 1000 +labels 1000 +jobs 1000 +job-events 1000 \ No newline at end of file diff --git a/tools/data_generators/rbac_dummy_data_generator.py b/tools/data_generators/rbac_dummy_data_generator.py index 42d02f6e56..046e4d851c 100755 --- a/tools/data_generators/rbac_dummy_data_generator.py +++ b/tools/data_generators/rbac_dummy_data_generator.py @@ -63,7 +63,7 @@ option_list = [ help='number of workflow job templates to create'), make_option('--nodes', action='store', type='int', default=200, help='number of workflow job template nodes to create'), - make_option('--labels', action='store', type='int', default=200, + make_option('--labels', action='store', type='int', default=100, help='labels to create, will associate 10x as many'), make_option('--jobs', action='store', type='int', default=200, help='number of job entries to create'), @@ -71,6 +71,8 @@ option_list = [ help='number of job event entries to create'), make_option('--pretend', action='store_true', help="Don't commit the data to the database"), + make_option('--preset', action='store', type='string', default='', + help="Preset data set to use"), make_option('--prefix', action='store', type='string', default='', help="Prefix generated names with this string"), #make_option('--spread-bias', action='store', type='string', default='exponential', @@ -81,6 +83,10 @@ options, remainder = parser.parse_args() options = vars(options) +if options['preset']: + pass + + n_organizations = int(options['organizations']) n_users = int(options['users']) n_teams = int(options['teams']) @@ -454,10 +460,13 @@ try: inventory=inventory, project=project, credential=next(credential_gen), - defaults=dict(created_by=next(creator_gen), - modified_by=next(modifier_gen)), + defaults=dict( + created_by=next(creator_gen), + modified_by=next(modifier_gen), + playbook="debug.yml"), **extra_kwargs ) + job_template._is_new = _ job_templates.append(job_template) inv_idx += 1 if project_idx == 0 and i == 0: @@ -546,7 +555,7 @@ try: sys.stdout.write('\r Assigning %d to %s: %d ' % (n, org.name, i + 1)) sys.stdout.flush() label, _ = Label.objects.get_or_create( - name='%s Label %d Org %d' % (prefix, label_id, org_idx), + name='%sL_%do%d' % (prefix, label_id, org_idx), organization=org, defaults=dict(created_by=next(creator_gen), modified_by=next(modifier_gen)) @@ -560,16 +569,31 @@ try: jt_idx = 0 for n in spread(n_labels*7, n_job_templates): jt = job_templates[jt_idx] + if not jt._is_new: + continue print(' Giving %d labels to %s JT' % (n, jt.name)) for i in range(n): jt.labels.add(next(label_gen)) jt_idx += 1 + print('# Adding labels to workflow job templates') + wfjt_idx = 0 + for n in spread(n_labels*3, n_wfjts): + wfjt = wfjts[wfjt_idx] + if not jt._is_new: + continue + print(' Giving %d labels to %s WFJT' % (n, wfjt.name)) + for i in range(n): + wfjt.labels.add(next(label_gen)) + wfjt_idx += 1 + print('# Creating %d jobs' % n_jobs) group_idx = 0 job_template_idx = 0 for n in spread(n_jobs, n_job_templates): job_template = job_templates[job_template_idx] + if not jt._is_new: + continue for i in range(n): sys.stdout.write('\r Assigning %d to %s: %d ' % (n, job_template.name, i+ 1)) sys.stdout.flush() @@ -578,15 +602,25 @@ try: job_stat = 'failed' elif len(jobs) % 11 == 0: job_stat = 'canceled' - job, _ = Job.objects.get_or_create( - job_template=job_template, status=job_stat) - job._is_new = _ + job, _ = Job.objects.create( + job_template=job_template, + status=job_stat, name=job_template.name, + project=job_template.project, inventory=job_template.inventory, + credential=job_template.credential, + cloud_credential=job_template.cloud_credential, + network_credential=job_template.network_credential + ) jobs.append(job) + if i == n: + job_template.last_job = job + if job_template.pk % 5 == 0: + job_template.current_job = job + job_template.save() with transaction.atomic(): if job_template.inventory: inv_groups = [g for g in job_template.inventory.groups.all()] - if len(inv_groups) and job._is_new: + if len(inv_groups): JobHostSummary.objects.bulk_create([ JobHostSummary( job=job, host=h, host_name=h.name, processed=1, @@ -603,19 +637,20 @@ try: job_idx = 0 for n in spread(n_job_events, n_jobs): job = jobs[job_idx] + if not job._is_new: + continue sys.stdout.write('\r Creating %d job events for job %d' % (n, job.id)) sys.stdout.flush() # Check if job already has events, for idempotence - if job._is_new: - JobEvent.objects.bulk_create([ - JobEvent( - created=now(), - modified=now(), - job=job, - event='runner_on_ok' - ) - for i in range(n) - ]) + JobEvent.objects.bulk_create([ + JobEvent( + created=now(), + modified=now(), + job=job, + event='runner_on_ok' + ) + for i in range(n) + ]) job_idx += 1 if n: print('') From 8c05d7e968f2b61cff12ee68ba852e699248cf55 Mon Sep 17 00:00:00 2001 From: AlanCoding Date: Tue, 3 Jan 2017 08:53:45 -0500 Subject: [PATCH 8/9] clean up boilerplate for reading preset data --- tools/data_generators/load_presets.py | 30 ++++++++++----------------- 1 file changed, 11 insertions(+), 19 deletions(-) diff --git a/tools/data_generators/load_presets.py b/tools/data_generators/load_presets.py index 34561e8fa6..7461dc067b 100644 --- a/tools/data_generators/load_presets.py +++ b/tools/data_generators/load_presets.py @@ -1,34 +1,26 @@ import os +preset = 'medium' + presets_filename = os.path.abspath(os.path.join( os.path.dirname(os.path.abspath(__file__)), 'presets.tsv')) with open(presets_filename) as f: - r = f.read() + text = f.read() -print r +split_lines = [line.split('\t') for line in text.split('\n')] +keys = split_lines[0][1:] -lines = r.split('\n') -keys = lines[0].split('\t')[1:] - -preset = 'medium' - -col = None -for i, key in enumerate(keys): - if key == preset: - col = i - break - -if col is None: +try: + col = keys.index(preset) +except ValueError: raise Exception('Preset %s data set not found, options are %s' % (preset, keys)) +options = {cols[0]: cols[col+1] for cols in split_lines} -options = {} - -for line in lines[1:]: - cols = line.split('\t') - options[cols[0]] = cols[i+1] +print ' text ' +print text print ' options ' print options From 67686f640be8a40a01292ba8b12afb4bf8e079c8 Mon Sep 17 00:00:00 2001 From: AlanCoding Date: Tue, 3 Jan 2017 09:09:43 -0500 Subject: [PATCH 9/9] finish implementation of data generator presets, fix flake8 --- Makefile | 3 +- tools/data_generators/load_presets.py | 26 ----------- .../rbac_dummy_data_generator.py | 44 +++++++++++++------ 3 files changed, 32 insertions(+), 41 deletions(-) delete mode 100644 tools/data_generators/load_presets.py diff --git a/Makefile b/Makefile index 41e2bb70cc..0679a11911 100644 --- a/Makefile +++ b/Makefile @@ -507,11 +507,12 @@ test_tox: test_jenkins : test_coverage # Make fake data +DATA_GEN_PRESET = "" bulk_data: @if [ "$(VENV_BASE)" ]; then \ . $(VENV_BASE)/tower/bin/activate; \ fi; \ - $(PYTHON) tools/data_generators/rbac_dummy_data_generator.py + $(PYTHON) tools/data_generators/rbac_dummy_data_generator.py --preset=$(DATA_GEN_PRESET) # l10n TASKS # -------------------------------------- diff --git a/tools/data_generators/load_presets.py b/tools/data_generators/load_presets.py deleted file mode 100644 index 7461dc067b..0000000000 --- a/tools/data_generators/load_presets.py +++ /dev/null @@ -1,26 +0,0 @@ -import os - -preset = 'medium' - -presets_filename = os.path.abspath(os.path.join( - os.path.dirname(os.path.abspath(__file__)), - 'presets.tsv')) - -with open(presets_filename) as f: - text = f.read() - -split_lines = [line.split('\t') for line in text.split('\n')] -keys = split_lines[0][1:] - -try: - col = keys.index(preset) -except ValueError: - raise Exception('Preset %s data set not found, options are %s' % (preset, keys)) - -options = {cols[0]: cols[col+1] for cols in split_lines} - -print ' text ' -print text - -print ' options ' -print options diff --git a/tools/data_generators/rbac_dummy_data_generator.py b/tools/data_generators/rbac_dummy_data_generator.py index 046e4d851c..dbbb5f4aa2 100755 --- a/tools/data_generators/rbac_dummy_data_generator.py +++ b/tools/data_generators/rbac_dummy_data_generator.py @@ -84,7 +84,22 @@ options = vars(options) if options['preset']: - pass + # Read the numbers of resources from presets file, if provided + presets_filename = os.path.abspath(os.path.join( + os.path.dirname(os.path.abspath(__file__)), 'presets.tsv')) + + with open(presets_filename) as f: + text = f.read() + + split_lines = [line.split('\t') for line in text.split('\n')] + keys = split_lines[0][1:] + + try: + col = keys.index(options['preset']) + except ValueError: + raise Exception('Preset "%s" dataset not found, options are %s' % (options['preset'], keys)) + + options.update({cols[0]: cols[col + 1] for cols in split_lines}) n_organizations = int(options['organizations']) @@ -163,6 +178,7 @@ class Rollback(Exception): def mock_save(self, *args, **kwargs): return super(PrimordialModel, self).save(*args, **kwargs) + PrimordialModel.save = mock_save @@ -334,17 +350,17 @@ try: project, _ = Project.objects.get_or_create( name='%s Project %d Org %d' % (prefix, project_id, org_idx), organization=org, - defaults=dict(created_by=next(creator_gen), - modified_by=next(modifier_gen), - scm_url='https://github.com/jlaska/ansible-playbooks.git', - scm_type='git', - playbook_files=[ - "check.yml", "debug-50.yml", "debug.yml", "debug2.yml", - "debug_extra_vars.yml", "dynamic_inventory.yml", - "environ_test.yml", "fail_unless.yml", "pass_unless.yml", - "pause.yml", "ping-20.yml", "ping.yml", - "setfact_50.yml", "vault.yml" - ]) + defaults=dict( + created_by=next(creator_gen), modified_by=next(modifier_gen), + scm_url='https://github.com/jlaska/ansible-playbooks.git', + scm_type='git', + playbook_files=[ + "check.yml", "debug-50.yml", "debug.yml", "debug2.yml", + "debug_extra_vars.yml", "dynamic_inventory.yml", + "environ_test.yml", "fail_unless.yml", "pass_unless.yml", + "pause.yml", "ping-20.yml", "ping.yml", + "setfact_50.yml", "vault.yml" + ]) ) projects.append(project) if org_idx == 0 and i == 0: @@ -567,7 +583,7 @@ try: print('# Adding labels to job templates') jt_idx = 0 - for n in spread(n_labels*7, n_job_templates): + for n in spread(n_labels * 7, n_job_templates): jt = job_templates[jt_idx] if not jt._is_new: continue @@ -578,7 +594,7 @@ try: print('# Adding labels to workflow job templates') wfjt_idx = 0 - for n in spread(n_labels*3, n_wfjts): + for n in spread(n_labels * 3, n_wfjts): wfjt = wfjts[wfjt_idx] if not jt._is_new: continue