Jobs and Sockets

Job submission now removes the spinner once the job is started. It is also 'socket aware'. In the event the socket connection is unavailable on SCM updates and inventory sync runs it prompts the user with a dialog letting them know to hit the refresh button. Came up with a way to 'monitor' socket connections. On the main connection and on the Jobs page connecgtions we now check the connection every 3 seconds. If the connection status is error, or if the status returns 'connnecting' 2x in a row, we kill and restart the connections.
This commit is contained in:
Chris Houseknecht
2014-07-18 19:59:28 -04:00
parent dde42b04f2
commit 2961d5150f
9 changed files with 152 additions and 449 deletions

View File

@@ -411,7 +411,7 @@ angular.module('Tower', [
function ($compile, $cookieStore, $rootScope, $log, CheckLicense, $location, Authorization, LoadBasePaths, ViewLicense,
Timer, ClearScope, HideStream, Socket) {
var e, html, sock;
var e, html, sock, checkCount;
LoadBasePaths();
@@ -468,13 +468,17 @@ angular.module('Tower', [
HideStream();
}
// remove any lingering intervals
if ($rootScope.jobDetailInterval) {
window.clearInterval($rootScope.jobDetailInterval);
}
if ($rootScope.jobStdOutInterval) {
window.clearInterval($rootScope.jobStdOutInterval);
}
if ($rootScope.checkSocketConnectionInterval) {
// use to monitor and restart socket connections
window.clearInterval($rootScope.checkSocketConnectionInterval);
}
// On each navigation request, check that the user is logged in
if (!/^\/(login|logout)/.test($location.path())) {
@@ -541,33 +545,37 @@ angular.module('Tower', [
function openSocket() {
sock = Socket({ scope: $rootScope, endpoint: "jobs" });
sock.init();
setTimeout(function() {
$rootScope.$apply(function() {
sock.checkStatus();
$log.debug('socket status: ' + $rootScope.socketStatus);
});
},2000);
sock.on("status_changed", function(data) {
$log.debug('Job ' + data.unified_job_id + ' status changed to ' + data.status);
$rootScope.$emit('JobStatusChange', data);
});
}
openSocket();
/*
$rootScope.socketToggle = function() {
switch($rootScope.socketStatus) {
case 'ok':
case 'connecting':
sock = null;
$rootScope.socketStatus = 'error';
$rootScope.socketTip = 'Disconnected. Click to connect.';
break;
case 'error':
sock = null;
$rootScope.socketStatus = '';
$rootScope.socketTip = '';
setTimeout(openSocket, 500);
setTimeout(function() {
$rootScope.$apply(function() {
sock.checkStatus();
$log.debug('socket status: ' + $rootScope.socketStatus);
});
},2000);
// monitor socket status
checkCount = 0;
setInterval(function() {
if (sock.checkStatus() === 'error' || checkCount > 2) {
// there's an error or we're stuck in a 'connecting' state. attempt to reconnect
sock = null;
$log.debug('attempting new socket connection');
openSocket();
checkCount = 0;
}
};*/
else if (sock.checkStatus() === 'connecting') {
checkCount++;
}
else {
checkCount = 0;
}
}, 3000);
}
]);

View File

@@ -14,7 +14,7 @@ var $AnsibleConfig = {
tooltip_delay: {show: 500, hide: 100}, // Default number of milliseconds to delay displaying/hiding tooltips
debug_mode: false, // Enable console logging messages
debug_mode: true, // Enable console logging messages
password_strength: 45, // User password strength. Integer between 0 and 100, 100 being impossibly strong.
// This value controls progress bar colors:

View File

@@ -10,8 +10,8 @@
'use strict';
function JobsListController ($log, $scope, $compile, $routeParams, ClearScope, Breadcrumbs, LoadBreadCrumbs, LoadSchedulesScope, LoadJobsScope, RunningJobsList, CompletedJobsList, QueuedJobsList,
ScheduledJobsList, GetChoices, GetBasePath, Wait, Socket) {
function JobsListController ($rootScope, $log, $scope, $compile, $routeParams, ClearScope, Breadcrumbs, LoadBreadCrumbs, LoadSchedulesScope,
LoadJobsScope, RunningJobsList, CompletedJobsList, QueuedJobsList, ScheduledJobsList, GetChoices, GetBasePath, Wait, Socket) {
ClearScope();
@@ -21,104 +21,70 @@ function JobsListController ($log, $scope, $compile, $routeParams, ClearScope, B
api_complete = false,
schedule_socket,
job_socket,
event_queue = [],
expecting = 0,
max_rows;
max_rows, checkCount;
job_socket = Socket({
scope: $scope,
endpoint: "jobs"
});
function openSockets() {
job_socket = Socket({
scope: $scope,
endpoint: "jobs"
});
job_socket.init();
job_socket.on("status_changed", function(data) {
if (api_complete) {
processEvent(data);
}
});
schedule_socket = Socket({
scope: $scope,
endpoint: "schedules"
});
schedule_socket.init();
schedule_socket.on("schedule_changed", function() {
if (api_complete) {
scheduled_scope.search('schedule');
}
});
}
openSockets();
job_socket.init();
job_socket.on("status_changed", function(data) {
if (api_complete) {
processEvent(data);
$rootScope.checkSocketConnectionInterval = setInterval(function() {
if (job_socket.checkStatus() === 'error' || checkCount > 2) {
// there's an error or we're stuck in a 'connecting' state. attempt to reconnect
$log.debug('jobs page: initializing and restarting socket connections');
job_socket = null;
schedule_socket = null;
openSockets();
checkCount = 0;
}
else if (job_socket.checkStatus() === 'connecting') {
checkCount++;
}
else {
event_queue.push(data);
checkCount = 0;
}
});
schedule_socket = Socket({
scope: $scope,
endpoint: "schedules"
});
schedule_socket.init();
schedule_socket.on("status_change", function() {
if (api_complete) {
scheduled_scope.search('schedule');
}
});
}, 3000);
function processEvent(event) {
expecting = 0;
switch(event.status) {
case 'running':
if (!inList(running_scope[RunningJobsList.name], event.unified_job_id)) {
expecting = 2;
running_scope.search('running_job');
queued_scope.search('queued_job');
}
running_scope.search('running_job');
queued_scope.search('queued_job');
break;
case 'new':
case 'pending':
case 'waiting':
if (!inList(queued_scope[QueuedJobsList.name], event.unified_job_id)) {
expecting = 1;
queued_scope.search('queued_job');
}
queued_scope.search('queued_job');
break;
case 'successful':
case 'failed':
case 'error':
case 'canceled':
if (!inList(completed_scope[CompletedJobsList.name], event.unified_job_id)) {
expecting = 2;
completed_scope.search('completed_job');
running_scope.search('running_job');
}
completed_scope.search('completed_job');
running_scope.search('running_job');
break;
}
}
function inList(list, id) {
var found = false;
list.every( function(row) {
if (row.id === id) {
found = true;
return false;
}
return true;
});
return found;
}
if ($scope.removeProcessQueue) {
$scope.removeProcessQueue();
}
$scope.removeProcessQueue = $scope.$on('ProcessQueue', function() {
var event;
listCount=0;
if (event_queue.length > 0) {
event = event_queue[0];
processEvent(event);
event_queue.splice(0,1);
if ($scope.removeListLoaded) {
$scope.removeListLoaded();
}
$scope.removeListLoaded = $scope.$on('listLoaded', function() {
listCount++;
if (listCount === expecting) {
$scope.$emit('ProcessQueue');
}
});
}
});
LoadBreadCrumbs();
if ($scope.removeListLoaded) {
@@ -128,7 +94,6 @@ function JobsListController ($log, $scope, $compile, $routeParams, ClearScope, B
listCount++;
if (listCount === 4) {
api_complete = true;
$scope.$emit('ProcessQueue');
}
});
@@ -286,280 +251,5 @@ function JobsListController ($log, $scope, $compile, $routeParams, ClearScope, B
}
}
JobsListController.$inject = [ '$log', '$scope', '$compile', '$routeParams', 'ClearScope', 'Breadcrumbs', 'LoadBreadCrumbs', 'LoadSchedulesScope', 'LoadJobsScope', 'RunningJobsList', 'CompletedJobsList',
'QueuedJobsList', 'ScheduledJobsList', 'GetChoices', 'GetBasePath', 'Wait', 'Socket'];
function JobsEdit($scope, $rootScope, $compile, $location, $log, $routeParams, JobForm, JobTemplateForm, GenerateForm, Rest,
Alert, ProcessErrors, LoadBreadCrumbs, RelatedSearchInit, RelatedPaginateInit, ReturnToCaller, ClearScope, InventoryList,
CredentialList, ProjectList, LookUpInit, PromptPasswords, GetBasePath, md5Setup, FormatDate, JobStatusToolTip, Wait, Empty,
ParseVariableString, GetChoices) {
ClearScope();
var defaultUrl = GetBasePath('jobs'),
generator = GenerateForm,
id = $routeParams.id,
loadingFinishedCount = 0,
templateForm = {},
choicesCount = 0;
generator.inject(JobForm, { mode: 'edit', related: true, scope: $scope });
$scope.job_id = id;
$scope.parseType = 'yaml';
$scope.statusSearchSpin = false;
$scope.disableParseSelection = true;
function getPlaybooks(project, playbook) {
if (!Empty(project)) {
var url = GetBasePath('projects') + project + '/playbooks/';
Rest.setUrl(url);
Rest.get()
.success(function (data) {
var i;
$scope.playbook_options = [];
for (i = 0; i < data.length; i++) {
$scope.playbook_options.push(data[i]);
}
for (i = 0; i < $scope.playbook_options.length; i++) {
if ($scope.playbook_options[i] === playbook) {
$scope.playbook = $scope.playbook_options[i];
}
}
$scope.$emit('jobTemplateLoadFinished');
})
.error(function () {
$scope.$emit('jobTemplateLoadFinished');
});
} else {
$scope.$emit('jobTemplateLoadFinished');
}
}
// Retrieve each related set and populate the playbook list
if ($scope.jobLoadedRemove) {
$scope.jobLoadedRemove();
}
$scope.jobLoadedRemove = $scope.$on('jobLoaded', function (e, related_cloud_credential, project, playbook) {
getPlaybooks(project, playbook);
if (related_cloud_credential) {
//Get the name of the cloud credential
Rest.setUrl(related_cloud_credential);
Rest.get()
.success(function (data) {
$scope.cloud_credential_name = data.name;
$scope.$emit('jobTemplateLoadFinished');
})
.error(function (data, status) {
ProcessErrors($scope, data, status, null, { hdr: 'Error!',
msg: 'Failed to related cloud credential. GET returned status: ' + status });
});
} else {
$scope.$emit('jobTemplateLoadFinished');
}
});
// Turn off 'Wait' after both cloud credential and playbook list come back
if ($scope.removeJobTemplateLoadFinished) {
$scope.removeJobTemplateLoadFinished();
}
$scope.removeJobTemplateLoadFinished = $scope.$on('jobTemplateLoadFinished', function () {
loadingFinishedCount++;
if (loadingFinishedCount >= 2) {
// The initial template load finished. Now load related jobs, which
// will turn off the 'working' spinner.
Wait('stop');
}
});
$scope.verbosity_options = [{
value: 0,
label: 'Default'
}, {
value: 1,
label: 'Verbose'
}, {
value: 3,
label: 'Debug'
}];
$scope.playbook_options = null;
$scope.playbook = null;
function calcRows(content) {
var n = content.match(/\n/g),
rows = (n) ? n.length : 1;
return (rows > 15) ? 15 : rows;
}
if ($scope.removeLoadJobTemplate) {
$scope.removeLoadJobTemplate();
}
$scope.removeLoadJobTemplate = $scope.$on('loadJobTemplate', function() {
// Retrieve the job detail record and prepopulate the form
Rest.setUrl(defaultUrl + ':id/');
Rest.get({ params: { id: id } })
.success(function (data) {
var i, fld;
LoadBreadCrumbs();
$scope.status = data.status;
$scope.created = FormatDate(data.created);
$scope.modified = FormatDate(data.modified);
$scope.result_stdout = data.result_stdout;
$scope.result_traceback = data.result_traceback;
$scope.stdout_rows = calcRows($scope.result_stdout);
$scope.traceback_rows = calcRows($scope.result_traceback);
$scope.job_explanation = data.job_explanation || 'Things may have ended badly or gone swimingly well';
// Now load the job template form
templateForm.addTitle = 'Create Job Templates';
templateForm.editTitle = '{{ name }}';
templateForm.name = 'job_templates';
templateForm.twoColumns = true;
templateForm.fields = angular.copy(JobTemplateForm.fields);
for (fld in templateForm.fields) {
templateForm.fields[fld].readonly = true;
}
if (data.type === "playbook_run") {
$('#ui-accordion-jobs-collapse-0-panel-1').empty();
generator.inject(templateForm, {
mode: 'edit',
id: 'ui-accordion-jobs-collapse-0-panel-1',
related: false,
scope: $scope,
breadCrumbs: false
});
}
else {
$('#ui-accordion-jobs-collapse-0-header-1').hide();
$('#ui-accordion-jobs-collapse-0-panel-1').empty().hide();
$('#jobs-collapse-0').accordion( "option", "collapsible", false );
}
for (fld in templateForm.fields) {
if (fld !== 'variables' && data[fld] !== null && data[fld] !== undefined) {
if (JobTemplateForm.fields[fld].type === 'select') {
if ($scope[fld + '_options'] && $scope[fld + '_options'].length > 0) {
for (i = 0; i < $scope[fld + '_options'].length; i++) {
if (data[fld] === $scope[fld + '_options'][i].value) {
$scope[fld] = $scope[fld + '_options'][i];
}
}
} else {
$scope[fld] = data[fld];
}
} else {
$scope[fld] = data[fld];
}
}
if (fld === 'variables') {
$scope.variables = ParseVariableString(data.extra_vars);
}
if (JobTemplateForm.fields[fld].type === 'lookup' && data.summary_fields[JobTemplateForm.fields[fld].sourceModel]) {
$scope[JobTemplateForm.fields[fld].sourceModel + '_' + JobTemplateForm.fields[fld].sourceField] =
data.summary_fields[JobTemplateForm.fields[fld].sourceModel][JobTemplateForm.fields[fld].sourceField];
}
}
$scope.id = data.id;
$scope.name = (data.summary_fields && data.summary_fields.job_template) ? data.summary_fields.job_template.name : '';
$scope.statusToolTip = JobStatusToolTip(data.status);
$scope.url = data.url;
$scope.project = data.project;
$scope.launch_type = data.launch_type;
// set the type
data.type = 'playbook_run'; //temporary
$scope.type_choices.every( function(choice) {
if (choice.value === data.type) {
$scope.type = choice.label;
return false;
}
return true;
});
$scope.$emit('jobLoaded', data.related.cloud_credential, data.project, data.playbook);
})
.error(function (data, status) {
ProcessErrors($scope, data, status, null, { hdr: 'Error!',
msg: 'Failed to retrieve job: ' + $routeParams.id + '. GET status: ' + status });
});
});
Wait('start');
if ($scope.removeChoicesReady) {
$scope.removeChoicesReady();
}
$scope.removeChoicesReady = $scope.$on('choicesReady', function() {
choicesCount++;
if (choicesCount === 2) {
$scope.$emit('loadJobTemplate');
}
});
GetChoices({
scope: $scope,
url: GetBasePath('unified_jobs'),
field: 'job_type',
variable: 'job_type_options',
callback: 'choicesReady'
});
/*GetChoices({
scope: $scope,
url: GetBasePath('jobs'),
field: 'status',
variable: 'status_choices',
callback: 'choicesReady'
});*/
GetChoices({
scope: $scope,
url: GetBasePath('unified_jobs'), //'/static/sample/data/types/data.json'
field: 'type',
variable: 'type_choices',
callback: 'choicesReady'
});
$scope.refresh = function () {
Wait('start');
Rest.setUrl(defaultUrl + id + '/');
Rest.get()
.success(function (data) {
$scope.status = data.status;
$scope.result_stdout = data.result_stdout;
$scope.result_traceback = data.result_traceback;
$scope.stdout_rows = calcRows($scope.result_stdout);
$scope.traceback_rows = calcRows($scope.result_traceback);
Wait('stop');
})
.error(function (data, status) {
ProcessErrors($scope, data, status, null, { hdr: 'Error!',
msg: 'Attempt to load job failed. GET returned status: ' + status });
});
};
$scope.jobSummary = function () {
$location.path('/jobs/' + id + '/job_host_summaries');
};
$scope.jobEvents = function () {
$location.path('/jobs/' + id + '/job_events');
};
}
JobsEdit.$inject = ['$scope', '$rootScope', '$compile', '$location', '$log', '$routeParams', 'JobForm', 'JobTemplateForm',
'GenerateForm', 'Rest', 'Alert', 'ProcessErrors', 'LoadBreadCrumbs', 'RelatedSearchInit', 'RelatedPaginateInit',
'ReturnToCaller', 'ClearScope', 'InventoryList', 'CredentialList', 'ProjectList', 'LookUpInit', 'PromptPasswords',
'GetBasePath', 'md5Setup', 'FormatDate', 'JobStatusToolTip', 'Wait', 'Empty', 'ParseVariableString', 'GetChoices',
'LoadDialogPartial'
];
JobsListController.$inject = ['$rootScope', '$log', '$scope', '$compile', '$routeParams', 'ClearScope', 'Breadcrumbs', 'LoadBreadCrumbs', 'LoadSchedulesScope', 'LoadJobsScope', 'RunningJobsList', 'CompletedJobsList',
'QueuedJobsList', 'ScheduledJobsList', 'GetChoices', 'GetBasePath', 'Wait', 'Socket'];

View File

@@ -63,7 +63,7 @@ function ProjectsList ($scope, $rootScope, $location, $log, $routeParams, Rest,
if (type.value === project.scm_type) {
$scope.projects[i].scm_type = type.label;
if (type.label === 'Manual') {
$scope.projects[i].scm_update_tooltip = 'Manaul projects do not require an SCM update';
$scope.projects[i].scm_update_tooltip = 'Manual projects do not require an SCM update';
$scope.projects[i].scm_schedule_tooltip = 'Manual projects do not require a schedule';
$scope.projects[i].scm_type_class = 'btn-disabled';
$scope.projects[i].statusTip = 'Not configured for SCM';
@@ -284,7 +284,7 @@ function ProjectsList ($scope, $rootScope, $location, $log, $routeParams, Rest,
Rest.setUrl(url);
Rest.post()
.success(function () {
Alert('SCM Update Cancel', 'Your request to cancel the update was submitted to the task maanger.', 'alert-info');
Alert('SCM Update Cancel', 'Your request to cancel the update was submitted to the task manager.', 'alert-info');
$scope.refresh();
})
.error(function (data, status) {

View File

@@ -551,15 +551,14 @@ function($location, Wait, GetBasePath, LookUpInit, JobTemplateForm, CredentialLi
}
scope.removeUpdateSubmitted = scope.$on('UpdateSubmitted', function() {
// Refresh the project list after update request submitted
if (scope.refreshJobs) {
Wait('stop');
Wait('stop');
if (scope.socketStatus === 'error') {
Alert('Update Started', 'The request to start the SCM update process was submitted. ' +
'To monitor the update status, refresh the page by clicking the <i class="fa fa-refresh"></i> button.', 'alert-info');
scope.refreshJobs();
'To monitor the update status, refresh the page by clicking the <i class="fa fa-refresh"></i> button.', 'alert-info');
if (scope.refresh) {
scope.refresh();
}
}
//else if (scope.refresh) {
// scope.refresh();
//}
});
if (scope.removePromptForPasswords) {
@@ -619,8 +618,17 @@ function($location, Wait, GetBasePath, LookUpInit, JobTemplateForm, CredentialLi
}
scope.removeUpdateSubmitted = scope.$on('UpdateSubmitted', function () {
Wait('stop');
// No need to do anything else. The caller should be connected to the socket server and
// handling real-time updates
if (scope.socketStatus === 'error') {
Alert('Sync Started', 'The request to start the inventory sync process was submitted. ' +
'To monitor the status refresh the page by clicking the <i class="fa fa-refresh"></i> button.', 'alert-info');
if (scope.refreshGroups) {
// inventory detail page
scope.refreshGroups();
}
else if (scope.refresh) {
scope.refresh();
}
}
});
if (scope.removePromptForPasswords) {

View File

@@ -127,15 +127,12 @@ angular.module('SocketIO', ['AuthService', 'Utilities'])
// Check connection status
var self = this;
if (self.socket.socket.connected) {
$log.debug('Socket connected');
self.scope.socketStatus = 'ok';
}
else if (self.socket.socket.connecting || self.socket.socket.reconnecting) {
$log.debug('Socket connecting...');
self.scope.socketStatus = 'connecting';
}
else {
$log.debug('Socket error: connection refused');
self.scope.socketStatus = 'error';
}
self.scope.socketTip = getSocketTip(self.scope.socketStatus);

View File

@@ -16,30 +16,30 @@ angular.module('Utilities', ['RestServices', 'Utilities'])
* Place to remove things that might be lingering from a prior tab or view.
* This used to destroy the scope, but that causes issues in angular 1.2.x
*/
.factory('ClearScope', [
function () {
return function () {
.factory('ClearScope', [ '$rootScope', function ($rootScope) {
return function () {
$('#form-modal .modal-body').empty();
$('#form-modal2 .modal-body').empty();
$rootScope.flashMessage = null;
$('.tooltip').each(function () {
$(this).remove();
});
$('#form-modal .modal-body').empty();
$('#form-modal2 .modal-body').empty();
$('.popover').each(function () {
$(this).remove();
});
$('.tooltip').each(function () {
$(this).remove();
});
try {
$('#help-modal').dialog('close');
} catch (e) {
// ignore
}
$(window).unbind('resize');
};
}
])
$('.popover').each(function () {
$(this).remove();
});
try {
$('#help-modal').dialog('close');
} catch (e) {
// ignore
}
$(window).unbind('resize');
};
}])
/* Empty()

File diff suppressed because one or more lines are too long

View File

@@ -436,7 +436,7 @@
<script src="{{ STATIC_URL }}lib/codemirror/addon/edit/matchbrackets.js"></script>
<script src="{{ STATIC_URL }}lib/codemirror/addon/selection/active-line.js"></script>
<script src="{{ STATIC_URL }}lib/scrollto/lib/jquery-scrollto.js"></script>
<script src="{{ STATIC_URL }}lib/socket.io-client/dist/socket.io.min.js"></script>
<script src="{{ STATIC_URL }}lib/socket.io-client/dist/socket.io.js"></script>
<script src="{{ STATIC_URL }}lib/d3js/build/d3.v3.min.js"></script>
<script src="{{ STATIC_URL }}lib/novus-nvd3/nv.d3.min.js"></script>
<script src="{{ STATIC_URL }}lib/d3Donut/d3Donut.js"></script>