Merge pull request #4644 from jlmitch5/jobResultsPerf

Job results performance updates
This commit is contained in:
jlmitch5 2017-01-12 16:15:16 -05:00 committed by GitHub
commit 2c3e05cbaa
11 changed files with 233 additions and 98 deletions

View File

@ -154,7 +154,7 @@ STDOUT_MAX_BYTES_DISPLAY = 1048576
# Returned in the header on event api lists as a recommendation to the UI
# on how many events to display before truncating/hiding
RECOMMENDED_MAX_EVENTS_DISPLAY_HEADER = 10000
RECOMMENDED_MAX_EVENTS_DISPLAY_HEADER = 4000
# The maximum size of the ansible callback event's res data structure
# beyond this limit and the value will be removed

View File

@ -15,7 +15,7 @@ export default [ 'templateUrl',
link: function(scope) {
// as count is changed by event data coming in,
// update the host status bar
scope.$watch('count', function(val) {
var toDestroy = scope.$watch('count', function(val) {
if (val) {
Object.keys(val).forEach(key => {
// reposition the hosts status bar by setting
@ -38,6 +38,10 @@ export default [ 'templateUrl',
.filter(key => (val[key] > 0)).length > 0);
}
});
scope.$on('$destroy', function(){
toDestroy();
});
}
};
}];

View File

@ -162,6 +162,7 @@
.JobResultsStdOut-stdoutColumn {
padding-left: 20px;
padding-right: 20px;
padding-top: 2px;
padding-bottom: 2px;
color: @default-interface-txt;
@ -171,6 +172,12 @@
width:100%;
}
.JobResultsStdOut-stdoutColumn--tooMany {
font-weight: bold;
text-transform: uppercase;
color: @default-err;
}
.JobResultsStdOut-stdoutColumn {
cursor: pointer;
}
@ -216,6 +223,11 @@
color: @default-interface-txt;
}
.JobResultsStdOut-cappedLine {
color: @b7grey;
font-style: italic;
}
@media (max-width: @breakpoint-md) {
.JobResultsStdOut-numberColumnPreload {
display: none;

View File

@ -12,6 +12,18 @@ export default [ 'templateUrl', '$timeout', '$location', '$anchorScroll',
templateUrl: templateUrl('job-results/job-results-stdout/job-results-stdout'),
restrict: 'E',
link: function(scope, element) {
var toDestroy = [],
resizer,
scrollWatcher;
scope.$on('$destroy', function(){
$(window).off("resize", resizer);
$(window).off("scroll", scrollWatcher);
$(".JobResultsStdOut-stdoutContainer").off('scroll',
scrollWatcher);
toDestroy.forEach(closureFunc => closureFunc());
});
scope.stdoutContainerAvailable.resolve("container available");
// utility function used to find the top visible line and
// parent header in the pane
@ -115,9 +127,15 @@ export default [ 'templateUrl', '$timeout', '$location', '$anchorScroll',
// stop iterating over the standard out
// lines once the first one has been
// found
$this = null;
return false;
}
});
}
$this = null;
});
$container = null;
return {
visLine: visItem,
@ -131,22 +149,24 @@ export default [ 'templateUrl', '$timeout', '$location', '$anchorScroll',
} else {
scope.isMobile = false;
}
// watch changes to the window size
$(window).resize(function() {
resizer = function() {
// and update the isMobile var accordingly
if (window.innerWidth <= 1200 && !scope.isMobile) {
scope.isMobile = true;
} else if (window.innerWidth > 1200 & scope.isMobile) {
scope.isMobile = false;
}
});
};
// watch changes to the window size
$(window).resize(resizer);
var lastScrollTop;
var initScrollTop = function() {
lastScrollTop = 0;
};
var scrollWatcher = function() {
scrollWatcher = function() {
var st = $(this).scrollTop();
var netScroll = st + $(this).innerHeight();
var fullHeight;
@ -178,11 +198,15 @@ export default [ 'templateUrl', '$timeout', '$location', '$anchorScroll',
}
lastScrollTop = st;
st = null;
netScroll = null;
fullHeight = null;
};
// update scroll watchers when isMobile changes based on
// window resize
scope.$watch('isMobile', function(val) {
toDestroy.push(scope.$watch('isMobile', function(val) {
if (val === true) {
// make sure ^ TOP always shown for mobile
scope.stdoutOverflowed = true;
@ -204,7 +228,7 @@ export default [ 'templateUrl', '$timeout', '$location', '$anchorScroll',
$(".JobResultsStdOut-stdoutContainer").on('scroll',
scrollWatcher);
}
});
}));
// called to scroll to follow anchor
scope.followScroll = function() {
@ -237,7 +261,7 @@ export default [ 'templateUrl', '$timeout', '$location', '$anchorScroll',
// if following becomes active, go ahead and get to the bottom
// of the standard out pane
scope.$watch('followEngaged', function(val) {
toDestroy.push(scope.$watch('followEngaged', function(val) {
// scroll to follow point if followEngaged is true
if (val) {
scope.followScroll();
@ -251,7 +275,7 @@ export default [ 'templateUrl', '$timeout', '$location', '$anchorScroll',
scope.followTooltip = "Click to follow standard out as it comes in.";
}
}
});
}));
// follow button ng-click function
scope.followToggleClicked = function() {

View File

@ -34,6 +34,13 @@
<div id="topAnchor" class="JobResultsStdOut-topAnchor"></div>
<div class="JobResultsStdOut-numberColumnPreload"></div>
<div id='lineAnchor' class="JobResultsStdOut-lineAnchor"></div>
<div class="JobResultsStdOut-aLineOfStdOut"
ng-show="tooManyEvents">
<div class="JobResultsStdOut-lineNumberColumn">
<span class="JobResultsStdOut-lineExpander"> </span>
</div>
<div class="JobResultsStdOut-stdoutColumn JobResultsStdOut-stdoutColumn--tooMany">The standard output is too large to display. Please specify additional filters to narrow the standard out.</div>
</div>
<div id="followAnchor"
class="JobResultsStdOut-followAnchor">
<div class="JobResultsStdOut-toTop"

View File

@ -148,6 +148,7 @@
background-color: @default-bg;
border-radius: 5px;
color: @default-interface-txt;
margin-right: -5px;
}
.JobResults-panelRight {

View File

@ -1,5 +1,11 @@
export default ['jobData', 'jobDataOptions', 'jobLabels', 'jobFinished', 'count', '$scope', 'ParseTypeChange', 'ParseVariableString', 'jobResultsService', 'eventQueue', '$compile', '$log', 'Dataset', '$q', 'Rest', '$state', 'QuerySet', '$rootScope', 'moment',
function(jobData, jobDataOptions, jobLabels, jobFinished, count, $scope, ParseTypeChange, ParseVariableString, jobResultsService, eventQueue, $compile, $log, Dataset, $q, Rest, $state, QuerySet, $rootScope, moment) {
var toDestroy = [];
var cancelRequests = false;
// this allows you to manage the timing of rest-call based events as
// filters are updated. see processPage for more info
var currentContext = 1;
// used for tag search
$scope.job_event_dataset = Dataset.data;
@ -66,14 +72,14 @@ function(jobData, jobDataOptions, jobLabels, jobFinished, count, $scope, ParseTy
// update label in left pane and tooltip in right pane when the job_status
// changes
$scope.$watch('job_status', function(status) {
toDestroy.push($scope.$watch('job_status', function(status) {
if (status) {
$scope.status_label = $scope.jobOptions.status.choices
.filter(val => val[0] === status)
.map(val => val[1])[0];
$scope.status_tooltip = "Job " + $scope.status_label;
}
});
}));
// update the job_status value. Use the cached rootScope value if there
// is one. This is a workaround when the rest call for the jobData is
@ -185,7 +191,12 @@ function(jobData, jobDataOptions, jobLabels, jobFinished, count, $scope, ParseTy
// This is where the async updates to the UI actually happen.
// Flow is event queue munging in the service -> $scope setting in here
var processEvent = function(event) {
var processEvent = function(event, context) {
// only care about filter context checking when the event comes
// as a rest call
if (context && context !== currentContext) {
return;
}
// put the event in the queue
var mungedEvent = eventQueue.populate(event);
@ -278,6 +289,9 @@ function(jobData, jobDataOptions, jobLabels, jobFinished, count, $scope, ParseTy
.stdout)($scope.events[mungedEvent
.counter]));
}
classList = null;
putIn = null;
} else {
// this is a header or recap line, so just
// append to the bottom
@ -357,99 +371,113 @@ function(jobData, jobDataOptions, jobLabels, jobFinished, count, $scope, ParseTy
getSkeleton(jobData.related.job_events + "?order_by=id&or__event__in=playbook_on_start,playbook_on_play_start,playbook_on_task_start,playbook_on_stats");
});
var getEvents;
var processPage = function(events, context) {
// currentContext is the context of the filter when this request
// to processPage was made
//
// currentContext is the context of the filter currently
//
// if they are not the same, make sure to stop process events/
// making rest calls for next pages/etc. (you can see context is
// also passed into getEvents and processEvent and similar checks
// exist in these functions)
//
// also, if the page doesn't contain results (i.e.: the response
// returns an error), don't process the page
if (context !== currentContext || events === undefined ||
events.results === undefined) {
return;
}
events.results.forEach(event => {
// get the name in the same format as the data
// coming over the websocket
event.event_name = event.event;
delete event.event;
processEvent(event, context);
});
if (events.next && !cancelRequests) {
getEvents(events.next, context);
} else {
// put those paused events into the pane
$scope.gotPreviouslyRanEvents.resolve("");
}
};
// grab non-header recap lines
var getEvents = function(url) {
getEvents = function(url, context) {
if (context !== currentContext) {
return;
}
jobResultsService.getEvents(url)
.then(events => {
events.results.forEach(event => {
// get the name in the same format as the data
// coming over the websocket
event.event_name = event.event;
delete event.event;
processEvent(event);
});
if (events.next) {
getEvents(events.next);
} else {
// put those paused events into the pane
$scope.gotPreviouslyRanEvents.resolve("");
}
processPage(events, context);
});
};
// grab non-header recap lines
$scope.$watch('job_event_dataset', function(val) {
toDestroy.push($scope.$watch('job_event_dataset', function(val) {
eventQueue.initialize();
Object.keys($scope.events)
.forEach(v => {
// dont destroy scope events for skeleton lines
let name = $scope.events[v].event.name;
if (!(name === "playbook_on_play_start" ||
name === "playbook_on_task_start" ||
name === "playbook_on_stats")) {
$scope.events[v].$destroy();
$scope.events[v] = null;
delete $scope.events[v];
}
});
// pause websocket events from coming in to the pane
$scope.gotPreviouslyRanEvents = $q.defer();
currentContext += 1;
let context = currentContext;
$( ".JobResultsStdOut-aLineOfStdOut.not_skeleton" ).remove();
$scope.hasSkeleton.promise.then(() => {
val.results.forEach(event => {
// get the name in the same format as the data
// coming over the websocket
event.event_name = event.event;
delete event.event;
processEvent(event);
});
if (val.next) {
getEvents(val.next);
if (val.count > parseInt(val.maxEvents)) {
$(".header_task").hide();
$(".header_play").hide();
$scope.tooManyEvents = true;
} else {
// put those paused events into the pane
$scope.gotPreviouslyRanEvents.resolve("");
$(".header_task").show();
$(".header_play").show();
$scope.tooManyEvents = false;
processPage(val, context);
}
});
});
}));
// Processing of job_events messages from the websocket
$scope.$on(`ws-job_events-${$scope.job.id}`, function(e, data) {
toDestroy.push($scope.$on(`ws-job_events-${$scope.job.id}`, function(e, data) {
$q.all([$scope.gotPreviouslyRanEvents.promise,
$scope.hasSkeleton.promise]).then(() => {
var url = Dataset
.config.url.split("?")[0] +
QuerySet.encodeQueryset($state.params.job_event_search);
var noFilter = (url.split("&")
.filter(v => v.indexOf("page=") !== 0 &&
v.indexOf("/api/v1") !== 0 &&
v.indexOf("order_by=id") !== 0 &&
v.indexOf("not__event__in=playbook_on_start,playbook_on_play_start,playbook_on_task_start,playbook_on_stats") !== 0).length === 0);
if(data.event_name === "playbook_on_start" ||
data.event_name === "playbook_on_play_start" ||
data.event_name === "playbook_on_task_start" ||
data.event_name === "playbook_on_stats" ||
noFilter) {
// for header and recap lines, as well as if no filters
// were added by the user, just put the line in the
// standard out pane (and increment play and task
// count)
if (data.event_name === "playbook_on_play_start") {
$scope.playCount++;
} else if (data.event_name === "playbook_on_task_start") {
$scope.taskCount++;
}
processEvent(data);
} else {
// to make sure host event/verbose lines go through a
// user defined filter, appent the id to the url, and
// make a request to the job_events endpoint with the
// id of the incoming event appended. If the event,
// is returned, put the line in the standard out pane
Rest.setUrl(`${url}&id=${data.id}`);
Rest.get()
.success(function(isHere) {
if (isHere.count) {
processEvent(data);
}
});
// put the line in the
// standard out pane (and increment play and task
// count if applicable)
if (data.event_name === "playbook_on_play_start") {
$scope.playCount++;
} else if (data.event_name === "playbook_on_task_start") {
$scope.taskCount++;
}
processEvent(data);
});
});
}));
// Processing of job-status messages from the websocket
$scope.$on(`ws-jobs`, function(e, data) {
toDestroy.push($scope.$on(`ws-jobs`, function(e, data) {
if (parseInt(data.unified_job_id, 10) ===
parseInt($scope.job.id,10)) {
// controller is defined, so set the job_status
@ -477,5 +505,19 @@ function(jobData, jobDataOptions, jobLabels, jobFinished, count, $scope, ParseTy
// for this job. cache the socket status on root scope
$rootScope['lastSocketStatus' + data.unified_job_id] = data.status;
}
}));
$scope.$on('$destroy', function(){
$( ".JobResultsStdOut-aLineOfStdOut" ).remove();
cancelRequests = true;
eventQueue.initialize();
Object.keys($scope.events)
.forEach(v => {
$scope.events[v].$destroy();
$scope.events[v] = null;
});
$scope.events = {};
clearInterval(elapsedInterval);
toDestroy.forEach(closureFunc => closureFunc());
});
}];

View File

@ -25,6 +25,7 @@ export default {
params: {
job_event_search: {
value: {
page_size: 100,
order_by: 'id',
not__event__in: 'playbook_on_start,playbook_on_play_start,playbook_on_task_start,playbook_on_stats'
},

View File

@ -27,6 +27,7 @@ export default ['$log', 'moment', function($log, moment){
line = line.replace(/u001b/g, '');
// ansi classes
line = line.replace(/\[1;im/g, '<span class="JobResultsStdOut-cappedLine">');
line = line.replace(/\[1;31m/g, '<span class="ansi1 ansi31">');
line = line.replace(/\[0;31m/g, '<span class="ansi1 ansi31">');
line = line.replace(/\[0;32m/g, '<span class="ansi32">');
@ -185,7 +186,6 @@ export default ['$log', 'moment', function($log, moment){
data-uuid="${clickClass}">
</i>
</span>`;
// console.log(expandDom);
return expandDom;
} else {
// non-header lines don't get an expander
@ -193,10 +193,22 @@ export default ['$log', 'moment', function($log, moment){
}
},
getLineArr: function(event) {
return _
.zip(_.range(event.start_line + 1,
event.end_line + 1),
event.stdout.replace("\t", " ").split("\r\n").slice(0, -1));
let lineNums = _.range(event.start_line + 1,
event.end_line + 1);
let lines = event.stdout
.replace("\t", " ")
.split("\r\n");
if (lineNums.length > lines.length) {
let padBy = lineNums.length - lines.length;
for (let i = 0; i <= padBy; i++) {
lines.push("[1;imLine capped.[0m");
}
}
return _.zip(lineNums, lines).slice(0, -1);
},
// public function that provides the parsed stdout line, given a
// job_event

View File

@ -238,21 +238,33 @@ export default ['$q', 'Rest', 'ProcessErrors', '$rootScope', 'Wait', 'DjangoSear
Wait('start');
this.url = `${endpoint}${this.encodeQueryset(params)}`;
Rest.setUrl(this.url);
return Rest.get()
.success(this.success.bind(this))
.error(this.error.bind(this))
.finally(Wait('stop'));
.then(function(response) {
Wait('stop');
if (response
.headers('X-UI-Max-Events') !== null) {
response.data.maxEvents = response.
headers('X-UI-Max-Events');
}
return response;
})
.catch(function(response) {
Wait('stop');
this.error(response.data, response.status);
return response;
}.bind(this));
},
error(data, status) {
ProcessErrors($rootScope, data, status, null, {
hdr: 'Error!',
msg: 'Call to ' + this.url + '. GET returned: ' + status
});
},
success(data) {
return data;
},
}
};
}
];

View File

@ -31,10 +31,14 @@ describe('parseStdoutService', () => {
unstyledLine = 'ok: [host-00]';
expect(parseStdoutService.prettify(line, unstyled)).toBe(unstyledLine);
});
it('can return empty strings', () => {
expect(parseStdoutService.prettify("")).toBe("");
});
});
describe('getLineClasses()', () => {
xit('creates a string that is used as a class', () => {
it('creates a string that is used as a class', () => {
let headerEvent = {
event_name: 'playbook_on_task_start',
event_data: {
@ -44,12 +48,15 @@ describe('parseStdoutService', () => {
};
let lineNum = 3;
let line = "TASK [setup] *******************************************************************";
let styledLine = " header_task header_task_80dd087c-268b-45e8-9aab-1083bcfd9364 play_0f667a23-d9ab-4128-a735-80566bcdbca0 line_num_3";
let styledLine = " header_task header_task_80dd087c-268b-45e8-9aab-1083bcfd9364 actual_header play_0f667a23-d9ab-4128-a735-80566bcdbca0 line_num_3";
expect(parseStdoutService.getLineClasses(headerEvent, line, lineNum)).toBe(styledLine);
});
});
describe('getStartTime()', () => {
// TODO: the problem is that the date here calls moment, and thus
// the date will be timezone'd in the string (this could be
// different based on where you are)
xit('creates returns a badge with the start time of the event', () => {
let headerEvent = {
event_name: 'playbook_on_play_start',
@ -115,6 +122,19 @@ describe('parseStdoutService', () => {
expect(returnedEvent).toEqual(expectedReturn);
});
it('deals correctly with capped lines', () => {
let mockEvent = {
start_line: 7,
end_line: 11,
stdout: "a\r\nb\r\nc..."
};
let expectedReturn = [[8, "a"],[9, "b"], [10,"c..."], [11, "[1;imLine capped.[0m"]];
let returnedEvent = parseStdoutService.getLineArr(mockEvent);
expect(returnedEvent).toEqual(expectedReturn);
});
});
describe('parseStdout()', () => {