be a bit more respectful of event request

minimum events made now is 10,000, used to be 10,000*number of cores.

could dig deeper but this is better for debugging changes to script
This commit is contained in:
Elijah DeLee 2020-05-27 19:34:12 -04:00
parent 5d5edf6535
commit 90ca2fd59b

View File

@ -219,8 +219,13 @@ def generate_events(events, job, time_delta):
cores = multiprocessing.cpu_count()
workers = []
for i in range(cores):
p = multiprocessing.Process(target=firehose, args=(job, events / cores, created_stamp, modified_stamp))
num_procs = min(cores, events)
num_events = events // num_procs
if num_events <= 1:
num_events = events
for i in range(num_procs):
p = multiprocessing.Process(target=firehose, args=(job, num_events, created_stamp, modified_stamp))
p.daemon = True
workers.append(p)