External-Mirrors/awx
2
0
Fork 0
mirror of https://github.com/ansible/awx.git synced 2026-02-28 08:18:43 -03:30
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #6033 from AlanCoding/job_batch

Browse Source 
Make job batch size configurable, fix _by fields

Reviewed-by: https://github.com/apps/softwarefactory-project-zuul
This commit is contained in:
softwarefactory-project-zuul[bot]
2020-02-21 21:22:58 +00:00
committed by GitHub
parent 2a8679234a 866dd6b259
commit ba3253e2e2
1 changed files with 9 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
tools/scripts/firehose.py
View File

@@ -121,7 +121,7 @@ def cleanup(sql):
conn.close() conn.close()
def generate_jobs(jobs): def generate_jobs(jobs, batch_size):
print(f'inserting {jobs} job(s)') print(f'inserting {jobs} job(s)')
sys.path.insert(0, pkg_resources.get_distribution('awx').module_path) sys.path.insert(0, pkg_resources.get_distribution('awx').module_path)
from awx import prepare_env from awx import prepare_env
@@ -150,7 +150,7 @@ def generate_jobs(jobs):
jt_defaults = dict( jt_defaults = dict(
(f.attname, getattr(jt, f.attname)) (f.attname, getattr(jt, f.attname))
for f in JobTemplate._meta.get_fields() for f in JobTemplate._meta.get_fields()
if f.editable and f.attname in job_field_names and getattr(jt, f.attname) if f.concrete and f.attname in job_field_names and getattr(jt, f.attname)
) )
jt_defaults['job_template_id'] = jt.pk jt_defaults['job_template_id'] = jt.pk
jt_defaults['unified_job_template_id'] = jt.pk # populated by save method jt_defaults['unified_job_template_id'] = jt.pk # populated by save method
@@ -176,10 +176,10 @@ def generate_jobs(jobs):
while jobs > 0: while jobs > 0:
s_loop = time() s_loop = time()
print('running batch {}, runtime {}'.format(i, time() - s)) print('running batch {}, runtime {}'.format(i, time() - s))
created, jt_pos = make_batch(min(jobs, 1000), jt_pos) created, jt_pos = make_batch(min(jobs, batch_size), jt_pos)
print('took {}'.format(time() - s_loop)) print('took {}'.format(time() - s_loop))
i += 1 i += 1
jobs -= 1000 jobs -= batch_size
return created return created
@@ -283,9 +283,13 @@ if __name__ == '__main__':
parser.add_argument( parser.add_argument(
'--events', type=int, help='Number of events to create.', '--events', type=int, help='Number of events to create.',
default=1000000000) # 1B by default default=1000000000) # 1B by default
parser.add_argument(
'--batch-size', type=int, help='Number of jobs to create in a single batch.',
default=1000)
params = parser.parse_args() params = parser.parse_args()
jobs = params.jobs jobs = params.jobs
events = params.events events = params.events
batch_size = params.batch_size
print(datetime.datetime.utcnow().isoformat()) print(datetime.datetime.utcnow().isoformat())
created = generate_jobs(jobs) created = generate_jobs(jobs, batch_size=batch_size)
generate_events(events, str(created.pk)) generate_events(events, str(created.pk))